diff options
Diffstat (limited to 'src/runtime')
358 files changed, 25075 insertions, 26658 deletions
diff --git a/src/runtime/alg.go b/src/runtime/alg.go index e9ed59503..e367bc5b2 100644 --- a/src/runtime/alg.go +++ b/src/runtime/alg.go @@ -314,9 +314,6 @@ const hashRandomBytes = 32 var aeskeysched [hashRandomBytes]byte -//go:noescape -func get_random_data(rnd *unsafe.Pointer, n *int32) - func init() { if theGoos == "nacl" { return diff --git a/src/runtime/arch1_386.go b/src/runtime/arch1_386.go new file mode 100644 index 000000000..a73e207ed --- /dev/null +++ b/src/runtime/arch1_386.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + thechar = '8' + _BigEndian = 0 + _CacheLineSize = 64 + _RuntimeGogoBytes = 64 + _PhysPageSize = goos_nacl*65536 + (1-goos_nacl)*4096 // 4k normally; 64k on NaCl + _PCQuantum = 1 + _Int64Align = 4 +) diff --git a/src/runtime/arch1_amd64.go b/src/runtime/arch1_amd64.go new file mode 100644 index 000000000..794b7f65c --- /dev/null +++ b/src/runtime/arch1_amd64.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + thechar = '6' + _BigEndian = 0 + _CacheLineSize = 64 + _RuntimeGogoBytes = 64 + (goos_plan9|goos_solaris|goos_windows)*16 + _PhysPageSize = 4096 + _PCQuantum = 1 + _Int64Align = 8 +) diff --git a/src/runtime/arch1_arm.go b/src/runtime/arch1_arm.go new file mode 100644 index 000000000..6662eaeac --- /dev/null +++ b/src/runtime/arch1_arm.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + thechar = '5' + _BigEndian = 0 + _CacheLineSize = 32 + _RuntimeGogoBytes = 60 + _PhysPageSize = 65536*goos_nacl + 4096*(1-goos_nacl) + _PCQuantum = 4 + _Int64Align = 4 +) diff --git a/src/runtime/arch1_power64.go b/src/runtime/arch1_power64.go new file mode 100644 index 000000000..01e2b70f9 --- /dev/null +++ b/src/runtime/arch1_power64.go @@ -0,0 +1,15 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + thechar = '9' + _BigEndian = 1 + _CacheLineSize = 64 + _RuntimeGogoBytes = 64 + _PhysPageSize = 65536 + _PCQuantum = 4 + _Int64Align = 8 +) diff --git a/src/runtime/arch1_power64le.go b/src/runtime/arch1_power64le.go new file mode 100644 index 000000000..6580732a3 --- /dev/null +++ b/src/runtime/arch1_power64le.go @@ -0,0 +1,15 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + thechar = '9' + _BigEndian = 0 + _CacheLineSize = 64 + _RuntimeGogoBytes = 64 + _PhysPageSize = 65536 + _PCQuantum = 4 + _Int64Align = 8 +) diff --git a/src/runtime/arch_386.h b/src/runtime/arch_386.h deleted file mode 100644 index 75a5ba77f..000000000 --- a/src/runtime/arch_386.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -enum { - thechar = '8', - BigEndian = 0, - CacheLineSize = 64, - RuntimeGogoBytes = 64, -#ifdef GOOS_nacl - PhysPageSize = 65536, -#else - PhysPageSize = 4096, -#endif - PCQuantum = 1, - Int64Align = 4 -}; diff --git a/src/runtime/arch_amd64.h b/src/runtime/arch_amd64.h deleted file mode 100644 index d7b81ee90..000000000 --- a/src/runtime/arch_amd64.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -enum { - thechar = '6', - BigEndian = 0, - CacheLineSize = 64, -#ifdef GOOS_solaris - RuntimeGogoBytes = 80, -#else -#ifdef GOOS_windows - RuntimeGogoBytes = 80, -#else -#ifdef GOOS_plan9 - RuntimeGogoBytes = 80, -#else - RuntimeGogoBytes = 64, -#endif // Plan 9 -#endif // Windows -#endif // Solaris - PhysPageSize = 4096, - PCQuantum = 1, - Int64Align = 8 -}; diff --git a/src/runtime/arch_arm.h b/src/runtime/arch_arm.h deleted file mode 100644 index 637a334a0..000000000 --- a/src/runtime/arch_arm.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -enum { - thechar = '5', - BigEndian = 0, - CacheLineSize = 32, - RuntimeGogoBytes = 60, -#ifdef GOOS_nacl - PhysPageSize = 65536, -#else - PhysPageSize = 4096, -#endif - PCQuantum = 4, - Int64Align = 4 -}; diff --git a/src/runtime/arch_power64.go b/src/runtime/arch_power64.go new file mode 100644 index 000000000..270cd7b95 --- /dev/null +++ b/src/runtime/arch_power64.go @@ -0,0 +1,8 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type uintreg uint64 +type intptr int64 // TODO(rsc): remove diff --git a/src/runtime/arch_power64le.go b/src/runtime/arch_power64le.go new file mode 100644 index 000000000..270cd7b95 --- /dev/null +++ b/src/runtime/arch_power64le.go @@ -0,0 +1,8 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type uintreg uint64 +type intptr int64 // TODO(rsc): remove diff --git a/src/runtime/asm.s b/src/runtime/asm.s index e6d782f37..f1c812b90 100644 --- a/src/runtime/asm.s +++ b/src/runtime/asm.s @@ -12,3 +12,8 @@ DATA runtime·no_pointers_stackmap+0x00(SB)/4, $2 DATA runtime·no_pointers_stackmap+0x04(SB)/4, $0 GLOBL runtime·no_pointers_stackmap(SB),RODATA, $8 +TEXT runtime·nop(SB),NOSPLIT,$0-0 + RET + +GLOBL runtime·mheap_(SB), NOPTR, $0 +GLOBL runtime·memstats(SB), NOPTR, $0 diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index b4b81d739..a02bb5556 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -49,7 +50,7 @@ nocpuinfo: // update stackguard after _cgo_init MOVL $runtime·g0(SB), CX MOVL (g_stack+stack_lo)(CX), AX - ADDL $const_StackGuard, AX + ADDL $const__StackGuard, AX MOVL AX, g_stackguard0(CX) MOVL AX, g_stackguard1(CX) @@ -199,62 +200,49 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-4 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-4 + MOVL fn+0(FP), DI // DI = fn get_tls(CX) MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m + MOVL m_gsignal(BX), DX // DX = gsignal CMPL AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVL fn+0(FP), DI // DI = fn - MOVL DI, DX - MOVL 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-4 - MOVL fn+0(FP), DI // DI = fn - get_tls(CX) - MOVL g(CX), AX // AX = g - MOVL g_m(AX), BX // BX = m + JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX - JEQ onm + JEQ noswitch MOVL m_curg(BX), BP CMPL AX, BP - JEQ oncurg + JEQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. + // Bad: g is not gsignal, not g0, not curg. What is it? // Hide call from linker nosplit analysis. - MOVL $runtime·badonm(SB), AX + MOVL $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVL $runtime·switchtoM(SB), (g_sched+gobuf_pc)(AX) + // be systemstack_switch if the G stack is scanned. + MOVL $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX) MOVL SP, (g_sched+gobuf_sp)(AX) MOVL AX, (g_sched+gobuf_g)(AX) // switch to g0 MOVL DX, g(CX) MOVL (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUBL $4, BX MOVL $runtime·mstart(SB), DX MOVL DX, 0(BX) @@ -275,8 +263,8 @@ oncurg: MOVL $0, (g_sched+gobuf_sp)(AX) RET -onm: - // already on m stack, just call directly +noswitch: + // already on system stack, just call directly MOVL DI, DX MOVL 0(DI), DI CALL DI @@ -486,11 +474,11 @@ TEXT runtime·cas64(SB), NOSPLIT, $0-21 MOVL new_hi+16(FP), CX LOCK CMPXCHG8B 0(BP) - JNZ cas64_fail + JNZ fail MOVL $1, AX MOVB AX, ret+20(FP) RET -cas64_fail: +fail: MOVL $0, AX MOVB AX, ret+20(FP) RET @@ -502,7 +490,7 @@ cas64_fail: // return 1; // }else // return 0; -TEXT runtime·casp(SB), NOSPLIT, $0-13 +TEXT runtime·casp1(SB), NOSPLIT, $0-13 MOVL ptr+0(FP), BX MOVL old+4(FP), AX MOVL new+8(FP), CX @@ -537,7 +525,7 @@ TEXT runtime·xchg(SB), NOSPLIT, $0-12 MOVL AX, ret+8(FP) RET -TEXT runtime·xchgp(SB), NOSPLIT, $0-12 +TEXT runtime·xchgp1(SB), NOSPLIT, $0-12 MOVL ptr+0(FP), BX MOVL new+4(FP), AX XCHGL AX, 0(BX) @@ -555,7 +543,7 @@ again: JNZ again RET -TEXT runtime·atomicstorep(SB), NOSPLIT, $0-8 +TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-8 MOVL ptr+0(FP), BX MOVL val+4(FP), AX XCHGL AX, 0(BX) @@ -740,7 +728,7 @@ needm: // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVL m_g0(BP), SI @@ -1356,29 +1344,29 @@ TEXT strings·IndexByte(SB),NOSPLIT,$0 // AX = 1/0/-1 TEXT runtime·cmpbody(SB),NOSPLIT,$0-0 CMPL SI, DI - JEQ cmp_allsame + JEQ allsame CMPL BX, DX MOVL DX, BP CMOVLLT BX, BP // BP = min(alen, blen) CMPL BP, $4 - JB cmp_small + JB small TESTL $0x4000000, runtime·cpuid_edx(SB) // check for sse2 - JE cmp_mediumloop -cmp_largeloop: + JE mediumloop +largeloop: CMPL BP, $16 - JB cmp_mediumloop + JB mediumloop MOVOU (SI), X0 MOVOU (DI), X1 PCMPEQB X0, X1 PMOVMSKB X1, AX XORL $0xffff, AX // convert EQ to NE - JNE cmp_diff16 // branch if at least one byte is not equal + JNE diff16 // branch if at least one byte is not equal ADDL $16, SI ADDL $16, DI SUBL $16, BP - JMP cmp_largeloop + JMP largeloop -cmp_diff16: +diff16: BSFL AX, BX // index of first byte that differs XORL AX, AX MOVB (SI)(BX*1), CX @@ -1387,25 +1375,25 @@ cmp_diff16: LEAL -1(AX*2), AX // convert 1/0 to +1/-1 RET -cmp_mediumloop: +mediumloop: CMPL BP, $4 - JBE cmp_0through4 + JBE _0through4 MOVL (SI), AX MOVL (DI), CX CMPL AX, CX - JNE cmp_diff4 + JNE diff4 ADDL $4, SI ADDL $4, DI SUBL $4, BP - JMP cmp_mediumloop + JMP mediumloop -cmp_0through4: +_0through4: MOVL -4(SI)(BP*1), AX MOVL -4(DI)(BP*1), CX CMPL AX, CX - JEQ cmp_allsame + JEQ allsame -cmp_diff4: +diff4: BSWAPL AX // reverse order of bytes BSWAPL CX XORL AX, CX // find bit differences @@ -1416,37 +1404,37 @@ cmp_diff4: RET // 0-3 bytes in common -cmp_small: +small: LEAL (BP*8), CX NEGL CX - JEQ cmp_allsame + JEQ allsame // load si CMPB SI, $0xfc - JA cmp_si_high + JA si_high MOVL (SI), SI - JMP cmp_si_finish -cmp_si_high: + JMP si_finish +si_high: MOVL -4(SI)(BP*1), SI SHRL CX, SI -cmp_si_finish: +si_finish: SHLL CX, SI // same for di CMPB DI, $0xfc - JA cmp_di_high + JA di_high MOVL (DI), DI - JMP cmp_di_finish -cmp_di_high: + JMP di_finish +di_high: MOVL -4(DI)(BP*1), DI SHRL CX, DI -cmp_di_finish: +di_finish: SHLL CX, DI BSWAPL SI // reverse order of bytes BSWAPL DI XORL SI, DI // find bit differences - JEQ cmp_allsame + JEQ allsame BSRL DI, CX // index of highest bit difference SHRL CX, SI // move a's bit to bottom ANDL $1, SI // mask bit @@ -1455,7 +1443,7 @@ cmp_di_finish: // all the bytes in common are the same, so we just need // to compare the lengths. -cmp_allsame: +allsame: XORL AX, AX XORL CX, CX CMPL BX, DX @@ -2290,3 +2278,10 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$0-4 + get_tls(CX) + MOVL g(CX), AX + MOVL AX, ret+0(FP) + RET + diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 39d7c78f2..6e3f5ff6c 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -47,7 +48,7 @@ nocpuinfo: // update stackguard after _cgo_init MOVQ $runtime·g0(SB), CX MOVQ (g_stack+stack_lo)(CX), AX - ADDQ $const_StackGuard, AX + ADDQ $const__StackGuard, AX MOVQ AX, g_stackguard0(CX) MOVQ AX, g_stackguard1(CX) @@ -189,55 +190,41 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-8 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-8 + MOVQ fn+0(FP), DI // DI = fn get_tls(CX) MOVQ g(CX), AX // AX = g MOVQ g_m(AX), BX // BX = m + MOVQ m_gsignal(BX), DX // DX = gsignal CMPQ AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVQ fn+0(FP), DI // DI = fn - MOVQ DI, DX - MOVQ 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-8 - MOVQ fn+0(FP), DI // DI = fn - get_tls(CX) - MOVQ g(CX), AX // AX = g - MOVQ g_m(AX), BX // BX = m + JEQ noswitch MOVQ m_g0(BX), DX // DX = g0 CMPQ AX, DX - JEQ onm + JEQ noswitch MOVQ m_curg(BX), BP CMPQ AX, BP - JEQ oncurg + JEQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. - // Hide call from linker nosplit analysis. - MOVQ $runtime·badonm(SB), AX + // Bad: g is not gsignal, not g0, not curg. What is it? + MOVQ $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVQ $runtime·switchtoM(SB), BP + // be systemstack_switch if the G stack is scanned. + MOVQ $runtime·systemstack_switch(SB), BP MOVQ BP, (g_sched+gobuf_pc)(AX) MOVQ SP, (g_sched+gobuf_sp)(AX) MOVQ AX, (g_sched+gobuf_g)(AX) @@ -245,7 +232,7 @@ oncurg: // switch to g0 MOVQ DX, g(CX) MOVQ (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUBQ $8, BX MOVQ $runtime·mstart(SB), DX MOVQ DX, 0(BX) @@ -266,7 +253,7 @@ oncurg: MOVQ $0, (g_sched+gobuf_sp)(AX) RET -onm: +noswitch: // already on m stack, just call directly MOVQ DI, DX MOVQ 0(DI), DI @@ -461,11 +448,11 @@ TEXT runtime·cas64(SB), NOSPLIT, $0-25 MOVQ new+16(FP), CX LOCK CMPXCHGQ CX, 0(BX) - JNZ cas64_fail + JNZ fail MOVL $1, AX MOVB AX, ret+24(FP) RET -cas64_fail: +fail: MOVL $0, AX MOVB AX, ret+24(FP) RET @@ -489,7 +476,7 @@ TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16 // return 1; // } else // return 0; -TEXT runtime·casp(SB), NOSPLIT, $0-25 +TEXT runtime·casp1(SB), NOSPLIT, $0-25 MOVQ ptr+0(FP), BX MOVQ old+8(FP), AX MOVQ new+16(FP), CX @@ -541,7 +528,7 @@ TEXT runtime·xchg64(SB), NOSPLIT, $0-24 MOVQ AX, ret+16(FP) RET -TEXT runtime·xchgp(SB), NOSPLIT, $0-24 +TEXT runtime·xchgp1(SB), NOSPLIT, $0-24 MOVQ ptr+0(FP), BX MOVQ new+8(FP), AX XCHGQ AX, 0(BX) @@ -559,7 +546,7 @@ again: JNZ again RET -TEXT runtime·atomicstorep(SB), NOSPLIT, $0-16 +TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-16 MOVQ ptr+0(FP), BX MOVQ val+8(FP), AX XCHGQ AX, 0(BX) @@ -726,7 +713,7 @@ needm: // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVQ m_g0(BP), SI @@ -890,24 +877,24 @@ TEXT runtime·aeshashbody(SB),NOSPLIT,$0-32 MOVO runtime·aeskeysched+0(SB), X2 MOVO runtime·aeskeysched+16(SB), X3 CMPQ CX, $16 - JB aessmall -aesloop: + JB small +loop: CMPQ CX, $16 - JBE aesloopend + JBE loopend MOVOU (AX), X1 AESENC X2, X0 AESENC X1, X0 SUBQ $16, CX ADDQ $16, AX - JMP aesloop + JMP loop // 1-16 bytes remaining -aesloopend: +loopend: // This load may overlap with the previous load above. // We'll hash some bytes twice, but that's ok. MOVOU -16(AX)(CX*1), X1 JMP partial // 0-15 bytes -aessmall: +small: TESTQ CX, CX JE finalize // 0 bytes @@ -1050,18 +1037,18 @@ TEXT runtime·eqstring(SB),NOSPLIT,$0-33 MOVQ s1len+8(FP), AX MOVQ s2len+24(FP), BX CMPQ AX, BX - JNE different + JNE noteq MOVQ s1str+0(FP), SI MOVQ s2str+16(FP), DI CMPQ SI, DI - JEQ same + JEQ eq CALL runtime·memeqbody(SB) MOVB AX, v+32(FP) RET -same: +eq: MOVB $1, v+32(FP) RET -different: +noteq: MOVB $0, v+32(FP) RET @@ -1184,29 +1171,29 @@ TEXT runtime·cmpbytes(SB),NOSPLIT,$0-56 // AX = 1/0/-1 TEXT runtime·cmpbody(SB),NOSPLIT,$0-0 CMPQ SI, DI - JEQ cmp_allsame + JEQ allsame CMPQ BX, DX MOVQ DX, BP CMOVQLT BX, BP // BP = min(alen, blen) = # of bytes to compare CMPQ BP, $8 - JB cmp_small + JB small -cmp_loop: +loop: CMPQ BP, $16 - JBE cmp_0through16 + JBE _0through16 MOVOU (SI), X0 MOVOU (DI), X1 PCMPEQB X0, X1 PMOVMSKB X1, AX XORQ $0xffff, AX // convert EQ to NE - JNE cmp_diff16 // branch if at least one byte is not equal + JNE diff16 // branch if at least one byte is not equal ADDQ $16, SI ADDQ $16, DI SUBQ $16, BP - JMP cmp_loop + JMP loop // AX = bit mask of differences -cmp_diff16: +diff16: BSFQ AX, BX // index of first byte that differs XORQ AX, AX MOVB (SI)(BX*1), CX @@ -1216,21 +1203,21 @@ cmp_diff16: RET // 0 through 16 bytes left, alen>=8, blen>=8 -cmp_0through16: +_0through16: CMPQ BP, $8 - JBE cmp_0through8 + JBE _0through8 MOVQ (SI), AX MOVQ (DI), CX CMPQ AX, CX - JNE cmp_diff8 -cmp_0through8: + JNE diff8 +_0through8: MOVQ -8(SI)(BP*1), AX MOVQ -8(DI)(BP*1), CX CMPQ AX, CX - JEQ cmp_allsame + JEQ allsame // AX and CX contain parts of a and b that differ. -cmp_diff8: +diff8: BSWAPQ AX // reverse order of bytes BSWAPQ CX XORQ AX, CX @@ -1241,44 +1228,44 @@ cmp_diff8: RET // 0-7 bytes in common -cmp_small: +small: LEAQ (BP*8), CX // bytes left -> bits left NEGQ CX // - bits lift (== 64 - bits left mod 64) - JEQ cmp_allsame + JEQ allsame // load bytes of a into high bytes of AX CMPB SI, $0xf8 - JA cmp_si_high + JA si_high MOVQ (SI), SI - JMP cmp_si_finish -cmp_si_high: + JMP si_finish +si_high: MOVQ -8(SI)(BP*1), SI SHRQ CX, SI -cmp_si_finish: +si_finish: SHLQ CX, SI // load bytes of b in to high bytes of BX CMPB DI, $0xf8 - JA cmp_di_high + JA di_high MOVQ (DI), DI - JMP cmp_di_finish -cmp_di_high: + JMP di_finish +di_high: MOVQ -8(DI)(BP*1), DI SHRQ CX, DI -cmp_di_finish: +di_finish: SHLQ CX, DI BSWAPQ SI // reverse order of bytes BSWAPQ DI XORQ SI, DI // find bit differences - JEQ cmp_allsame + JEQ allsame BSRQ DI, CX // index of highest bit difference SHRQ CX, SI // move a's bit to bottom ANDQ $1, SI // mask bit LEAQ -1(SI*2), AX // 1/0 => +1/-1 RET -cmp_allsame: +allsame: XORQ AX, AX XORQ CX, CX CMPQ BX, DX @@ -1313,7 +1300,7 @@ TEXT runtime·indexbytebody(SB),NOSPLIT,$0 MOVQ SI, DI CMPQ BX, $16 - JLT indexbyte_small + JLT small // round up to first 16-byte boundary TESTQ $15, SI @@ -1371,7 +1358,7 @@ failure: RET // handle for lengths < 16 -indexbyte_small: +small: MOVQ BX, CX REPN; SCASB JZ success @@ -2235,3 +2222,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$0-8 + get_tls(CX) + MOVQ g(CX), AX + MOVQ AX, ret+0(FP) + RET diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index a1116b5d4..b8370efd3 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -164,55 +165,42 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-4 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-4 + MOVL fn+0(FP), DI // DI = fn get_tls(CX) MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m + MOVL m_gsignal(BX), DX // DX = gsignal CMPL AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVL fn+0(FP), DI // DI = fn - MOVL DI, DX - MOVL 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-4 - MOVL fn+0(FP), DI // DI = fn - get_tls(CX) - MOVL g(CX), AX // AX = g - MOVL g_m(AX), BX // BX = m + JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX - JEQ onm + JEQ noswitch MOVL m_curg(BX), R8 CMPL AX, R8 - JEQ oncurg + JEQ switch // Not g0, not curg. Must be gsignal, but that's not allowed. // Hide call from linker nosplit analysis. - MOVL $runtime·badonm(SB), AX + MOVL $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVL $runtime·switchtoM(SB), SI + // be systemstack_switch if the G stack is scanned. + MOVL $runtime·systemstack_switch(SB), SI MOVL SI, (g_sched+gobuf_pc)(AX) MOVL SP, (g_sched+gobuf_sp)(AX) MOVL AX, (g_sched+gobuf_g)(AX) @@ -236,7 +224,7 @@ oncurg: MOVL $0, (g_sched+gobuf_sp)(AX) RET -onm: +noswitch: // already on m stack, just call directly MOVL DI, DX MOVL 0(DI), DI @@ -444,11 +432,11 @@ TEXT runtime·cas64(SB), NOSPLIT, $0-25 MOVQ new+16(FP), CX LOCK CMPXCHGQ CX, 0(BX) - JNZ cas64_fail + JNZ fail MOVL $1, AX MOVB AX, ret+24(FP) RET -cas64_fail: +fail: MOVL $0, AX MOVB AX, ret+24(FP) RET @@ -460,7 +448,7 @@ cas64_fail: // return 1; // } else // return 0; -TEXT runtime·casp(SB), NOSPLIT, $0-17 +TEXT runtime·casp1(SB), NOSPLIT, $0-17 MOVL ptr+0(FP), BX MOVL old+4(FP), AX MOVL new+8(FP), CX @@ -512,7 +500,7 @@ TEXT runtime·xchg64(SB), NOSPLIT, $0-24 MOVQ AX, ret+16(FP) RET -TEXT runtime·xchgp(SB), NOSPLIT, $0-12 +TEXT runtime·xchgp1(SB), NOSPLIT, $0-12 MOVL ptr+0(FP), BX MOVL new+4(FP), AX XCHGL AX, 0(BX) @@ -530,7 +518,7 @@ again: JNZ again RET -TEXT runtime·atomicstorep(SB), NOSPLIT, $0-8 +TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-8 MOVL ptr+0(FP), BX MOVL val+4(FP), AX XCHGL AX, 0(BX) @@ -834,29 +822,29 @@ TEXT runtime·cmpbytes(SB),NOSPLIT,$0-28 // AX = 1/0/-1 TEXT runtime·cmpbody(SB),NOSPLIT,$0-0 CMPQ SI, DI - JEQ cmp_allsame + JEQ allsame CMPQ BX, DX MOVQ DX, R8 CMOVQLT BX, R8 // R8 = min(alen, blen) = # of bytes to compare CMPQ R8, $8 - JB cmp_small + JB small -cmp_loop: +loop: CMPQ R8, $16 - JBE cmp_0through16 + JBE _0through16 MOVOU (SI), X0 MOVOU (DI), X1 PCMPEQB X0, X1 PMOVMSKB X1, AX XORQ $0xffff, AX // convert EQ to NE - JNE cmp_diff16 // branch if at least one byte is not equal + JNE diff16 // branch if at least one byte is not equal ADDQ $16, SI ADDQ $16, DI SUBQ $16, R8 - JMP cmp_loop + JMP loop // AX = bit mask of differences -cmp_diff16: +diff16: BSFQ AX, BX // index of first byte that differs XORQ AX, AX ADDQ BX, SI @@ -868,23 +856,23 @@ cmp_diff16: RET // 0 through 16 bytes left, alen>=8, blen>=8 -cmp_0through16: +_0through16: CMPQ R8, $8 - JBE cmp_0through8 + JBE _0through8 MOVQ (SI), AX MOVQ (DI), CX CMPQ AX, CX - JNE cmp_diff8 -cmp_0through8: + JNE diff8 +_0through8: ADDQ R8, SI ADDQ R8, DI MOVQ -8(SI), AX MOVQ -8(DI), CX CMPQ AX, CX - JEQ cmp_allsame + JEQ allsame // AX and CX contain parts of a and b that differ. -cmp_diff8: +diff8: BSWAPQ AX // reverse order of bytes BSWAPQ CX XORQ AX, CX @@ -895,46 +883,46 @@ cmp_diff8: RET // 0-7 bytes in common -cmp_small: +small: LEAQ (R8*8), CX // bytes left -> bits left NEGQ CX // - bits lift (== 64 - bits left mod 64) - JEQ cmp_allsame + JEQ allsame // load bytes of a into high bytes of AX CMPB SI, $0xf8 - JA cmp_si_high + JA si_high MOVQ (SI), SI - JMP cmp_si_finish -cmp_si_high: + JMP si_finish +si_high: ADDQ R8, SI MOVQ -8(SI), SI SHRQ CX, SI -cmp_si_finish: +si_finish: SHLQ CX, SI // load bytes of b in to high bytes of BX CMPB DI, $0xf8 - JA cmp_di_high + JA di_high MOVQ (DI), DI - JMP cmp_di_finish -cmp_di_high: + JMP di_finish +di_high: ADDQ R8, DI MOVQ -8(DI), DI SHRQ CX, DI -cmp_di_finish: +di_finish: SHLQ CX, DI BSWAPQ SI // reverse order of bytes BSWAPQ DI XORQ SI, DI // find bit differences - JEQ cmp_allsame + JEQ allsame BSRQ DI, CX // index of highest bit difference SHRQ CX, SI // move a's bit to bottom ANDQ $1, SI // mask bit LEAQ -1(SI*2), AX // 1/0 => +1/-1 RET -cmp_allsame: +allsame: XORQ AX, AX XORQ CX, CX CMPQ BX, DX @@ -969,7 +957,7 @@ TEXT runtime·indexbytebody(SB),NOSPLIT,$0 MOVL SI, DI CMPL BX, $16 - JLT indexbyte_small + JLT small // round up to first 16-byte boundary TESTL $15, SI @@ -1027,7 +1015,7 @@ failure: RET // handle for lengths < 16 -indexbyte_small: +small: MOVL BX, CX REPN; SCASB JZ success @@ -1085,3 +1073,9 @@ TEXT runtime·return0(SB), NOSPLIT, $0 TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$0-4 + get_tls(CX) + MOVL g(CX), AX + MOVL AX, ret+0(FP) + RET diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 0f3b5eeb8..583c7ba50 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -54,7 +55,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$-4 nocgo: // update stackguard after _cgo_init MOVW (g_stack+stack_lo)(g), R0 - ADD $const_StackGuard, R0 + ADD $const__StackGuard, R0 MOVW R0, g_stackguard0(g) MOVW R0, g_stackguard1(g) @@ -190,53 +191,42 @@ TEXT runtime·mcall(SB),NOSPLIT,$-4-4 B runtime·badmcall2(SB) RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB),NOSPLIT,$0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0 MOVW $0, R0 BL (R0) // clobber lr to ensure push {lr} is kept RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $-4-4 - MOVW g_m(g), R1 - MOVW m_gsignal(R1), R2 - CMP g, R2 - B.EQ ongsignal - B runtime·onM(SB) - -ongsignal: - MOVW fn+0(FP), R0 - MOVW R0, R7 - MOVW 0(R0), R0 - BL (R0) - RET - -// func onM(fn func()) -TEXT runtime·onM(SB),NOSPLIT,$0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB),NOSPLIT,$0-4 MOVW fn+0(FP), R0 // R0 = fn MOVW g_m(g), R1 // R1 = m + MOVW m_gsignal(R1), R2 // R2 = gsignal + CMP g, R2 + B.EQ noswitch + MOVW m_g0(R1), R2 // R2 = g0 CMP g, R2 - B.EQ onm + B.EQ noswitch MOVW m_curg(R1), R3 CMP g, R3 - B.EQ oncurg + B.EQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. + // Bad: g is not gsignal, not g0, not curg. What is it? // Hide call from linker nosplit analysis. - MOVW $runtime·badonm(SB), R0 + MOVW $runtime·badsystemstack(SB), R0 BL (R0) -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVW $runtime·switchtoM(SB), R3 + // be systemstack_switch if the G stack is scanned. + MOVW $runtime·systemstack_switch(SB), R3 ADD $4, R3, R3 // get past push {lr} MOVW R3, (g_sched+gobuf_pc)(g) MOVW SP, (g_sched+gobuf_sp)(g) @@ -249,7 +239,7 @@ oncurg: BL setg<>(SB) MOVW R5, R0 MOVW (g_sched+gobuf_sp)(R2), R3 - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUB $4, R3, R3 MOVW $runtime·mstart(SB), R4 MOVW R4, 0(R3) @@ -269,7 +259,7 @@ oncurg: MOVW R3, (g_sched+gobuf_sp)(g) RET -onm: +noswitch: MOVW R0, R7 MOVW 0(R0), R0 BL (R0) @@ -492,7 +482,7 @@ TEXT asmcgocall<>(SB),NOSPLIT,$0-0 MOVW g_m(g), R8 MOVW m_g0(R8), R3 CMP R3, g - BEQ asmcgocall_g0 + BEQ g0 BL gosave<>(SB) MOVW R0, R5 MOVW R3, R0 @@ -501,7 +491,7 @@ TEXT asmcgocall<>(SB),NOSPLIT,$0-0 MOVW (g_sched+gobuf_sp)(g), R13 // Now on a scheduling stack (a pthread-created stack). -asmcgocall_g0: +g0: SUB $24, R13 BIC $0x7, R13 // alignment for gcc ABI MOVW R4, 20(R13) // save old g @@ -564,7 +554,7 @@ TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-12 // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVW g_m(g), R8 @@ -751,13 +741,13 @@ TEXT runtime·memeq(SB),NOSPLIT,$-4-13 ADD R1, R3, R6 MOVW $1, R0 MOVB R0, ret+12(FP) -_next2: +loop: CMP R1, R6 RET.EQ MOVBU.P 1(R1), R4 MOVBU.P 1(R2), R5 CMP R4, R5 - BEQ _next2 + BEQ loop MOVW $0, R0 MOVB R0, ret+12(FP) @@ -780,13 +770,13 @@ TEXT runtime·eqstring(SB),NOSPLIT,$-4-17 CMP R2, R3 RET.EQ ADD R2, R0, R6 -_eqnext: +loop: CMP R2, R6 RET.EQ MOVBU.P 1(R2), R4 MOVBU.P 1(R3), R5 CMP R4, R5 - BEQ _eqnext + BEQ loop MOVB R7, v+16(FP) RET @@ -801,26 +791,26 @@ TEXT bytes·Equal(SB),NOSPLIT,$0 MOVW b_len+16(FP), R3 CMP R1, R3 // unequal lengths are not equal - B.NE _notequal + B.NE notequal MOVW a+0(FP), R0 MOVW b+12(FP), R2 ADD R0, R1 // end -_byteseq_next: +loop: CMP R0, R1 - B.EQ _equal // reached the end + B.EQ equal // reached the end MOVBU.P 1(R0), R4 MOVBU.P 1(R2), R5 CMP R4, R5 - B.EQ _byteseq_next + B.EQ loop -_notequal: +notequal: MOVW $0, R0 MOVBU R0, ret+24(FP) RET -_equal: +equal: MOVW $1, R0 MOVBU R0, ret+24(FP) RET @@ -1326,3 +1316,7 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$8 TEXT runtime·goexit(SB),NOSPLIT,$-4-0 MOVW R0, R0 // NOP BL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$-4-4 + MOVW g, ret+0(FP) + RET diff --git a/src/runtime/asm_power64x.s b/src/runtime/asm_power64x.s new file mode 100644 index 000000000..3f2ab6d0e --- /dev/null +++ b/src/runtime/asm_power64x.s @@ -0,0 +1,979 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le + +#include "go_asm.h" +#include "go_tls.h" +#include "funcdata.h" +#include "textflag.h" + +TEXT runtime·rt0_go(SB),NOSPLIT,$0 + // initialize essential registers + BL runtime·reginit(SB) + + SUB $24, R1 + MOVW R3, 8(R1) // argc + MOVD R4, 16(R1) // argv + + // create istack out of the given (operating system) stack. + // _cgo_init may update stackguard. + MOVD $runtime·g0(SB), g + MOVD $(-64*1024), R31 + ADD R31, R1, R3 + MOVD R3, g_stackguard0(g) + MOVD R3, g_stackguard1(g) + MOVD R3, (g_stack+stack_lo)(g) + MOVD R1, (g_stack+stack_hi)(g) + + // TODO: if there is a _cgo_init, call it. + // TODO: add TLS + + // set the per-goroutine and per-mach "registers" + MOVD $runtime·m0(SB), R3 + + // save m->g0 = g0 + MOVD g, m_g0(R3) + // save m0 to g0->m + MOVD R3, g_m(g) + + BL runtime·check(SB) + + // args are already prepared + BL runtime·args(SB) + BL runtime·osinit(SB) + BL runtime·schedinit(SB) + + // create a new goroutine to start program + MOVD $runtime·main·f(SB), R3 // entry + MOVDU R3, -8(R1) + MOVDU R0, -8(R1) + MOVDU R0, -8(R1) + BL runtime·newproc(SB) + ADD $24, R1 + + // start this M + BL runtime·mstart(SB) + + MOVD R0, 1(R0) + RETURN + +DATA runtime·main·f+0(SB)/8,$runtime·main(SB) +GLOBL runtime·main·f(SB),RODATA,$8 + +TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 + MOVD R0, 2(R0) // TODO: TD + RETURN + +TEXT runtime·asminit(SB),NOSPLIT,$-8-0 + RETURN + +TEXT runtime·reginit(SB),NOSPLIT,$-8-0 + // set R0 to zero, it's expected by the toolchain + XOR R0, R0 + // initialize essential FP registers + FMOVD $4503601774854144.0, F27 + FMOVD $0.5, F29 + FSUB F29, F29, F28 + FADD F29, F29, F30 + FADD F30, F30, F31 + RETURN + +/* + * go-routine + */ + +// void gosave(Gobuf*) +// save state in Gobuf; setjmp +TEXT runtime·gosave(SB), NOSPLIT, $-8-8 + MOVD buf+0(FP), R3 + MOVD R1, gobuf_sp(R3) + MOVD LR, R31 + MOVD R31, gobuf_pc(R3) + MOVD g, gobuf_g(R3) + MOVD R0, gobuf_lr(R3) + MOVD R0, gobuf_ret(R3) + MOVD R0, gobuf_ctxt(R3) + RETURN + +// void gogo(Gobuf*) +// restore state from Gobuf; longjmp +TEXT runtime·gogo(SB), NOSPLIT, $-8-8 + MOVD buf+0(FP), R5 + MOVD gobuf_g(R5), g // make sure g is not nil + MOVD 0(g), R4 + MOVD gobuf_sp(R5), R1 + MOVD gobuf_lr(R5), R31 + MOVD R31, LR + MOVD gobuf_ret(R5), R3 + MOVD gobuf_ctxt(R5), R11 + MOVD R0, gobuf_sp(R5) + MOVD R0, gobuf_ret(R5) + MOVD R0, gobuf_lr(R5) + MOVD R0, gobuf_ctxt(R5) + CMP R0, R0 // set condition codes for == test, needed by stack split + MOVD gobuf_pc(R5), R31 + MOVD R31, CTR + BR (CTR) + +// void mcall(fn func(*g)) +// Switch to m->g0's stack, call fn(g). +// Fn must never return. It should gogo(&g->sched) +// to keep running g. +TEXT runtime·mcall(SB), NOSPLIT, $-8-8 + // Save caller state in g->sched + MOVD R1, (g_sched+gobuf_sp)(g) + MOVD LR, R31 + MOVD R31, (g_sched+gobuf_pc)(g) + MOVD R0, (g_sched+gobuf_lr)(g) + MOVD g, (g_sched+gobuf_g)(g) + + // Switch to m->g0 & its stack, call fn. + MOVD g, R3 + MOVD g_m(g), R8 + MOVD m_g0(R8), g + CMP g, R3 + BNE 2(PC) + BR runtime·badmcall(SB) + MOVD fn+0(FP), R11 // context + MOVD 0(R11), R4 // code pointer + MOVD R4, CTR + MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp + MOVDU R3, -8(R1) + MOVDU R0, -8(R1) + BL (CTR) + BR runtime·badmcall2(SB) + +// systemstack_switch is a dummy routine that systemstack leaves at the bottom +// of the G stack. We need to distinguish the routine that +// lives at the bottom of the G stack from the one that lives +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 + UNDEF + BL (LR) // make sure this function is not leaf + RETURN + +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-8 + MOVD fn+0(FP), R3 // R3 = fn + MOVD R3, R11 // context + MOVD g_m(g), R4 // R4 = m + + MOVD m_gsignal(R4), R5 // R5 = gsignal + CMP g, R5 + BEQ noswitch + + MOVD m_g0(R4), R5 // R5 = g0 + CMP g, R5 + BEQ noswitch + + MOVD m_curg(R4), R6 + CMP g, R6 + BEQ switch + + // Bad: g is not gsignal, not g0, not curg. What is it? + // Hide call from linker nosplit analysis. + MOVD $runtime·badsystemstack(SB), R3 + MOVD R3, CTR + BL (CTR) + +switch: + // save our state in g->sched. Pretend to + // be systemstack_switch if the G stack is scanned. + MOVD $runtime·systemstack_switch(SB), R6 + ADD $8, R6 // get past prologue + MOVD R6, (g_sched+gobuf_pc)(g) + MOVD R1, (g_sched+gobuf_sp)(g) + MOVD R0, (g_sched+gobuf_lr)(g) + MOVD g, (g_sched+gobuf_g)(g) + + // switch to g0 + MOVD R5, g + MOVD (g_sched+gobuf_sp)(g), R3 + // make it look like mstart called systemstack on g0, to stop traceback + SUB $8, R3 + MOVD $runtime·mstart(SB), R4 + MOVD R4, 0(R3) + MOVD R3, R1 + + // call target function + MOVD 0(R11), R3 // code pointer + MOVD R3, CTR + BL (CTR) + + // switch back to g + MOVD g_m(g), R3 + MOVD m_curg(R3), g + MOVD (g_sched+gobuf_sp)(g), R1 + MOVD R0, (g_sched+gobuf_sp)(g) + RETURN + +noswitch: + // already on m stack, just call directly + MOVD 0(R11), R3 // code pointer + MOVD R3, CTR + BL (CTR) + RETURN + +/* + * support for morestack + */ + +// Called during function prolog when more stack is needed. +// Caller has already loaded: +// R3: framesize, R4: argsize, R5: LR +// +// The traceback routines see morestack on a g0 as being +// the top of a stack (for example, morestack calling newstack +// calling the scheduler calling newm calling gc), so we must +// record an argument size. For that purpose, it has no arguments. +TEXT runtime·morestack(SB),NOSPLIT,$-8-0 + // Cannot grow scheduler stack (m->g0). + MOVD g_m(g), R7 + MOVD m_g0(R7), R8 + CMP g, R8 + BNE 2(PC) + BL runtime·abort(SB) + + // Cannot grow signal stack (m->gsignal). + MOVD m_gsignal(R7), R8 + CMP g, R8 + BNE 2(PC) + BL runtime·abort(SB) + + // Called from f. + // Set g->sched to context in f. + MOVD R11, (g_sched+gobuf_ctxt)(g) + MOVD R1, (g_sched+gobuf_sp)(g) + MOVD LR, R8 + MOVD R8, (g_sched+gobuf_pc)(g) + MOVD R5, (g_sched+gobuf_lr)(g) + + // Called from f. + // Set m->morebuf to f's caller. + MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC + MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP + MOVD g, (m_morebuf+gobuf_g)(R7) + + // Call newstack on m->g0's stack. + MOVD m_g0(R7), g + MOVD (g_sched+gobuf_sp)(g), R1 + BL runtime·newstack(SB) + + // Not reached, but make sure the return PC from the call to newstack + // is still in this function, and not the beginning of the next. + UNDEF + +TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0 + MOVD R0, R11 + BR runtime·morestack(SB) + +// reflectcall: call a function with the given argument list +// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). +// we don't have variable-sized frames, so we use a small number +// of constant-sized-frame functions to encode a few bits of size in the pc. +// Caution: ugly multiline assembly macros in your future! + +#define DISPATCH(NAME,MAXSIZE) \ + MOVD $MAXSIZE, R31; \ + CMP R3, R31; \ + BGT 4(PC); \ + MOVD $NAME(SB), R31; \ + MOVD R31, CTR; \ + BR (CTR) +// Note: can't just "BR NAME(SB)" - bad inlining results. + +TEXT ·reflectcall(SB), NOSPLIT, $-8-24 + MOVWZ n+16(FP), R3 + DISPATCH(runtime·call16, 16) + DISPATCH(runtime·call32, 32) + DISPATCH(runtime·call64, 64) + DISPATCH(runtime·call128, 128) + DISPATCH(runtime·call256, 256) + DISPATCH(runtime·call512, 512) + DISPATCH(runtime·call1024, 1024) + DISPATCH(runtime·call2048, 2048) + DISPATCH(runtime·call4096, 4096) + DISPATCH(runtime·call8192, 8192) + DISPATCH(runtime·call16384, 16384) + DISPATCH(runtime·call32768, 32768) + DISPATCH(runtime·call65536, 65536) + DISPATCH(runtime·call131072, 131072) + DISPATCH(runtime·call262144, 262144) + DISPATCH(runtime·call524288, 524288) + DISPATCH(runtime·call1048576, 1048576) + DISPATCH(runtime·call2097152, 2097152) + DISPATCH(runtime·call4194304, 4194304) + DISPATCH(runtime·call8388608, 8388608) + DISPATCH(runtime·call16777216, 16777216) + DISPATCH(runtime·call33554432, 33554432) + DISPATCH(runtime·call67108864, 67108864) + DISPATCH(runtime·call134217728, 134217728) + DISPATCH(runtime·call268435456, 268435456) + DISPATCH(runtime·call536870912, 536870912) + DISPATCH(runtime·call1073741824, 1073741824) + MOVD $runtime·badreflectcall(SB), R31 + MOVD R31, CTR + BR (CTR) + +#define CALLFN(NAME,MAXSIZE) \ +TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ + NO_LOCAL_POINTERS; \ + /* copy arguments to stack */ \ + MOVD arg+8(FP), R3; \ + MOVWZ n+16(FP), R4; \ + MOVD R1, R5; \ + ADD $(8-1), R5; \ + SUB $1, R3; \ + ADD R5, R4; \ + CMP R5, R4; \ + BEQ 4(PC); \ + MOVBZU 1(R3), R6; \ + MOVBZU R6, 1(R5); \ + BR -4(PC); \ + /* call function */ \ + MOVD f+0(FP), R11; \ + MOVD (R11), R31; \ + MOVD R31, CTR; \ + PCDATA $PCDATA_StackMapIndex, $0; \ + BL (CTR); \ + /* copy return values back */ \ + MOVD arg+8(FP), R3; \ + MOVWZ n+16(FP), R4; \ + MOVWZ retoffset+20(FP), R6; \ + MOVD R1, R5; \ + ADD R6, R5; \ + ADD R6, R3; \ + SUB R6, R4; \ + ADD $(8-1), R5; \ + SUB $1, R3; \ + ADD R5, R4; \ + CMP R5, R4; \ + BEQ 4(PC); \ + MOVBZU 1(R5), R6; \ + MOVBZU R6, 1(R3); \ + BR -4(PC); \ + RETURN + +CALLFN(·call16, 16) +CALLFN(·call32, 32) +CALLFN(·call64, 64) +CALLFN(·call128, 128) +CALLFN(·call256, 256) +CALLFN(·call512, 512) +CALLFN(·call1024, 1024) +CALLFN(·call2048, 2048) +CALLFN(·call4096, 4096) +CALLFN(·call8192, 8192) +CALLFN(·call16384, 16384) +CALLFN(·call32768, 32768) +CALLFN(·call65536, 65536) +CALLFN(·call131072, 131072) +CALLFN(·call262144, 262144) +CALLFN(·call524288, 524288) +CALLFN(·call1048576, 1048576) +CALLFN(·call2097152, 2097152) +CALLFN(·call4194304, 4194304) +CALLFN(·call8388608, 8388608) +CALLFN(·call16777216, 16777216) +CALLFN(·call33554432, 33554432) +CALLFN(·call67108864, 67108864) +CALLFN(·call134217728, 134217728) +CALLFN(·call268435456, 268435456) +CALLFN(·call536870912, 536870912) +CALLFN(·call1073741824, 1073741824) + +// bool cas(uint32 *ptr, uint32 old, uint32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime·cas(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R3 + MOVWZ old+8(FP), R4 + MOVWZ new+12(FP), R5 +cas_again: + SYNC + LWAR (R3), R6 + CMPW R6, R4 + BNE cas_fail + STWCCC R5, (R3) + BNE cas_again + MOVD $1, R3 + SYNC + ISYNC + MOVB R3, ret+16(FP) + RETURN +cas_fail: + MOVD $0, R3 + BR -5(PC) + +// bool runtime·cas64(uint64 *ptr, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime·cas64(SB), NOSPLIT, $0-25 + MOVD ptr+0(FP), R3 + MOVD old+8(FP), R4 + MOVD new+16(FP), R5 +cas64_again: + SYNC + LDAR (R3), R6 + CMP R6, R4 + BNE cas64_fail + STDCCC R5, (R3) + BNE cas64_again + MOVD $1, R3 + SYNC + ISYNC + MOVB R3, ret+24(FP) + RETURN +cas64_fail: + MOVD $0, R3 + BR -5(PC) + +TEXT runtime·casuintptr(SB), NOSPLIT, $0-25 + BR runtime·cas64(SB) + +TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $-8-16 + BR runtime·atomicload64(SB) + +TEXT runtime·atomicloaduint(SB), NOSPLIT, $-8-16 + BR runtime·atomicload64(SB) + +TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16 + BR runtime·atomicstore64(SB) + +// bool casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime·casp1(SB), NOSPLIT, $0-25 + BR runtime·cas64(SB) + +// uint32 xadd(uint32 volatile *ptr, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime·xadd(SB), NOSPLIT, $0-20 + MOVD ptr+0(FP), R4 + MOVW delta+8(FP), R5 + SYNC + LWAR (R4), R3 + ADD R5, R3 + STWCCC R3, (R4) + BNE -4(PC) + SYNC + ISYNC + MOVW R3, ret+16(FP) + RETURN + +TEXT runtime·xadd64(SB), NOSPLIT, $0-24 + MOVD ptr+0(FP), R4 + MOVD delta+8(FP), R5 + SYNC + LDAR (R4), R3 + ADD R5, R3 + STDCCC R3, (R4) + BNE -4(PC) + SYNC + ISYNC + MOVD R3, ret+16(FP) + RETURN + +TEXT runtime·xchg(SB), NOSPLIT, $0-20 + MOVD ptr+0(FP), R4 + MOVW new+8(FP), R5 + SYNC + LWAR (R4), R3 + STWCCC R5, (R4) + BNE -3(PC) + SYNC + ISYNC + MOVW R3, ret+16(FP) + RETURN + +TEXT runtime·xchg64(SB), NOSPLIT, $0-24 + MOVD ptr+0(FP), R4 + MOVD new+8(FP), R5 + SYNC + LDAR (R4), R3 + STDCCC R5, (R4) + BNE -3(PC) + SYNC + ISYNC + MOVD R3, ret+16(FP) + RETURN + +TEXT runtime·xchgp1(SB), NOSPLIT, $0-24 + BR runtime·xchg64(SB) + +TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24 + BR runtime·xchg64(SB) + +TEXT runtime·procyield(SB),NOSPLIT,$0-0 + RETURN + +TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-16 + BR runtime·atomicstore64(SB) + +TEXT runtime·atomicstore(SB), NOSPLIT, $0-12 + MOVD ptr+0(FP), R3 + MOVW val+8(FP), R4 + SYNC + MOVW R4, 0(R3) + RETURN + +TEXT runtime·atomicstore64(SB), NOSPLIT, $0-16 + MOVD ptr+0(FP), R3 + MOVD val+8(FP), R4 + SYNC + MOVD R4, 0(R3) + RETURN + +// void runtime·atomicor8(byte volatile*, byte); +TEXT runtime·atomicor8(SB), NOSPLIT, $0-9 + MOVD ptr+0(FP), R3 + MOVBZ val+8(FP), R4 + // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R5 = (R3 << 0) & ~3 + RLDCR $0, R3, $~3, R5 + // Compute val shift. +#ifdef GOARCH_power64 + // Big endian. ptr = ptr ^ 3 + XOR $3, R3 +#endif + // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) + RLDC $3, R3, $(3*8), R6 + // Shift val for aligned ptr. R4 = val << R6 + SLD R6, R4, R4 + +atomicor8_again: + SYNC + LWAR (R5), R6 + OR R4, R6 + STWCCC R6, (R5) + BNE atomicor8_again + SYNC + ISYNC + RETURN + +// void jmpdefer(fv, sp); +// called from deferreturn. +// 1. grab stored LR for caller +// 2. sub 4 bytes to get back to BL deferreturn +// 3. BR to fn +TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 + MOVD 0(R1), R31 + SUB $4, R31 + MOVD R31, LR + + MOVD fv+0(FP), R11 + MOVD argp+8(FP), R1 + SUB $8, R1 + MOVD 0(R11), R3 + MOVD R3, CTR + BR (CTR) + +// Save state of caller into g->sched. Smashes R31. +TEXT gosave<>(SB),NOSPLIT,$-8 + MOVD LR, R31 + MOVD R31, (g_sched+gobuf_pc)(g) + MOVD R1, (g_sched+gobuf_sp)(g) + MOVD R0, (g_sched+gobuf_lr)(g) + MOVD R0, (g_sched+gobuf_ret)(g) + MOVD R0, (g_sched+gobuf_ctxt)(g) + RETURN + +// asmcgocall(void(*fn)(void*), void *arg) +// Call fn(arg) on the scheduler stack, +// aligned appropriately for the gcc ABI. +// See cgocall.c for more details. +TEXT ·asmcgocall(SB),NOSPLIT,$0-16 + MOVD R0, 21(R0) + +// cgocallback(void (*fn)(void*), void *frame, uintptr framesize) +// Turn the fn into a Go func (by taking its address) and call +// cgocallback_gofunc. +TEXT runtime·cgocallback(SB),NOSPLIT,$24-24 + MOVD R0, 22(R0) + +// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize) +// See cgocall.c for more details. +TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-24 + MOVD R0, 23(R0) + +// void setg(G*); set g. for use by needm. +TEXT runtime·setg(SB), NOSPLIT, $0-8 + MOVD R0, 24(R0) + +// void setg_gcc(G*); set g called from gcc. +TEXT setg_gcc<>(SB),NOSPLIT,$0 + MOVD R0, 25(R0) + +TEXT runtime·getcallerpc(SB),NOSPLIT,$-8-16 + MOVD 0(R1), R3 + MOVD R3, ret+8(FP) + RETURN + +TEXT runtime·gogetcallerpc(SB),NOSPLIT,$-8-16 + MOVD 0(R1), R3 + MOVD R3,ret+8(FP) + RETURN + +TEXT runtime·setcallerpc(SB),NOSPLIT,$-8-16 + MOVD pc+8(FP), R3 + MOVD R3, 0(R1) // set calling pc + RETURN + +TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 + MOVD argp+0(FP), R3 + SUB $8, R3 + MOVD R3, ret+8(FP) + RETURN + +// func gogetcallersp(p unsafe.Pointer) uintptr +TEXT runtime·gogetcallersp(SB),NOSPLIT,$0-16 + MOVD sp+0(FP), R3 + SUB $8, R3 + MOVD R3,ret+8(FP) + RETURN + +TEXT runtime·abort(SB),NOSPLIT,$-8-0 + MOVW (R0), R0 + UNDEF + +#define TBRL 268 +#define TBRU 269 /* Time base Upper/Lower */ + +// int64 runtime·cputicks(void) +TEXT runtime·cputicks(SB),NOSPLIT,$0-8 + MOVW SPR(TBRU), R4 + MOVW SPR(TBRL), R3 + MOVW SPR(TBRU), R5 + CMPW R4, R5 + BNE -4(PC) + SLD $32, R5 + OR R5, R3 + MOVD R3, ret+0(FP) + RETURN + +// AES hashing not implemented for Power +TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 + MOVW (R0), R1 +TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 + MOVW (R0), R1 +TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 + MOVW (R0), R1 +TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 + MOVW (R0), R1 + +TEXT runtime·memeq(SB),NOSPLIT,$-8-25 + MOVD a+0(FP), R3 + MOVD b+8(FP), R4 + MOVD size+16(FP), R5 + SUB $1, R3 + SUB $1, R4 + ADD R3, R5, R8 +loop: + CMP R3, R8 + BNE test + MOVD $1, R3 + MOVB R3, ret+24(FP) + RETURN +test: + MOVBZU 1(R3), R6 + MOVBZU 1(R4), R7 + CMP R6, R7 + BEQ loop + + MOVB R0, ret+24(FP) + RETURN + +// eqstring tests whether two strings are equal. +// See runtime_test.go:eqstring_generic for +// equivalent Go code. +TEXT runtime·eqstring(SB),NOSPLIT,$0-33 + MOVD s1len+8(FP), R4 + MOVD s2len+24(FP), R5 + CMP R4, R5 + BNE noteq + + MOVD s1str+0(FP), R3 + MOVD s2str+16(FP), R4 + SUB $1, R3 + SUB $1, R4 + ADD R3, R5, R8 +loop: + CMP R3, R8 + BNE 4(PC) + MOVD $1, R3 + MOVB R3, ret+32(FP) + RETURN + MOVBZU 1(R3), R6 + MOVBZU 1(R4), R7 + CMP R6, R7 + BEQ loop +noteq: + MOVB R0, ret+32(FP) + RETURN + +// TODO: share code with memeq? +TEXT bytes·Equal(SB),NOSPLIT,$0-49 + MOVD a_len+8(FP), R3 + MOVD b_len+32(FP), R4 + + CMP R3, R4 // unequal lengths are not equal + BNE noteq + + MOVD a+0(FP), R5 + MOVD b+24(FP), R6 + SUB $1, R5 + SUB $1, R6 + ADD R5, R3 // end-1 + +loop: + CMP R5, R3 + BEQ equal // reached the end + MOVBZU 1(R5), R4 + MOVBZU 1(R6), R7 + CMP R4, R7 + BEQ loop + +noteq: + MOVBZ R0, ret+48(FP) + RETURN + +equal: + MOVD $1, R3 + MOVBZ R3, ret+48(FP) + RETURN + +TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 + MOVD s+0(FP), R3 + MOVD s_len+8(FP), R4 + MOVBZ c+24(FP), R5 // byte to find + MOVD R3, R6 // store base for later + SUB $1, R3 + ADD R3, R4 // end-1 + +loop: + CMP R3, R4 + BEQ notfound + MOVBZU 1(R3), R7 + CMP R7, R5 + BNE loop + + SUB R6, R3 // remove base + MOVD R3, ret+32(FP) + RETURN + +notfound: + MOVD $-1, R3 + MOVD R3, ret+32(FP) + RETURN + +TEXT strings·IndexByte(SB),NOSPLIT,$0 + MOVD p+0(FP), R3 + MOVD b_len+8(FP), R4 + MOVBZ c+16(FP), R5 // byte to find + MOVD R3, R6 // store base for later + SUB $1, R3 + ADD R3, R4 // end-1 + +loop: + CMP R3, R4 + BEQ notfound + MOVBZU 1(R3), R7 + CMP R7, R5 + BNE loop + + SUB R6, R3 // remove base + MOVD R3, ret+24(FP) + RETURN + +notfound: + MOVD $-1, R3 + MOVD R3, ret+24(FP) + RETURN + + +// A Duff's device for zeroing memory. +// The compiler jumps to computed addresses within +// this routine to zero chunks of memory. Do not +// change this code without also changing the code +// in ../../cmd/9g/ggen.c:/^clearfat. +// R0: always zero +// R3 (aka REGRT1): ptr to memory to be zeroed - 8 +// On return, R3 points to the last zeroed dword. +TEXT runtime·duffzero(SB), NOSPLIT, $-8-0 + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + MOVDU R0, 8(R3) + RETURN + +TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 + MOVD g_m(g), R4 + MOVWZ m_fastrand(R4), R3 + ADD R3, R3 + CMPW R3, $0 + BGE 2(PC) + XOR $0x88888eef, R3 + MOVW R3, m_fastrand(R4) + MOVW R3, ret+0(FP) + RETURN + +TEXT runtime·return0(SB), NOSPLIT, $0 + MOVW $0, R3 + RETURN + +// Called from cgo wrappers, this function returns g->m->curg.stack.hi. +// Must obey the gcc calling convention. +TEXT _cgo_topofstack(SB),NOSPLIT,$0 + MOVD R0, 26(R0) + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$-8-0 + MOVD R0, R0 // NOP + BL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$-8-8 + MOVD g, ret+0(FP) + RETURN diff --git a/src/runtime/atomic_386.c b/src/runtime/atomic_386.c deleted file mode 100644 index 82d36f2d9..000000000 --- a/src/runtime/atomic_386.c +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "textflag.h" - -#pragma textflag NOSPLIT -uint32 -runtime·atomicload(uint32 volatile* addr) -{ - return *addr; -} - -#pragma textflag NOSPLIT -void* -runtime·atomicloadp(void* volatile* addr) -{ - return *addr; -} - -#pragma textflag NOSPLIT -uint64 -runtime·xadd64(uint64 volatile* addr, int64 v) -{ - uint64 old; - - do - old = *addr; - while(!runtime·cas64(addr, old, old+v)); - - return old+v; -} - -#pragma textflag NOSPLIT -uint64 -runtime·xchg64(uint64 volatile* addr, uint64 v) -{ - uint64 old; - - do - old = *addr; - while(!runtime·cas64(addr, old, v)); - - return old; -} diff --git a/src/runtime/atomic_386.go b/src/runtime/atomic_386.go new file mode 100644 index 000000000..5563432ef --- /dev/null +++ b/src/runtime/atomic_386.go @@ -0,0 +1,91 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// The calls to nop are to keep these functions from being inlined. +// If they are inlined we have no guarantee that later rewrites of the +// code by optimizers will preserve the relative order of memory accesses. + +//go:nosplit +func atomicload(ptr *uint32) uint32 { + nop() + return *ptr +} + +//go:nosplit +func atomicloadp(ptr unsafe.Pointer) unsafe.Pointer { + nop() + return *(*unsafe.Pointer)(ptr) +} + +//go:nosplit +func xadd64(ptr *uint64, delta int64) uint64 { + for { + old := *ptr + if cas64(ptr, old, old+uint64(delta)) { + return old + uint64(delta) + } + } +} + +//go:nosplit +func xchg64(ptr *uint64, new uint64) uint64 { + for { + old := *ptr + if cas64(ptr, old, new) { + return old + } + } +} + +//go:noescape +func xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func xchg(ptr *uint32, new uint32) uint32 + +// xchgp cannot have a go:noescape annotation, because +// while ptr does not escape, new does. If new is marked as +// not escaping, the compiler will make incorrect escape analysis +// decisions about the value being xchg'ed. +// Instead, make xchgp a wrapper around the actual atomic. +// When calling the wrapper we mark ptr as noescape explicitly. + +//go:nosplit +func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { + return xchgp1(noescape(ptr), new) +} + +func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer + +//go:noescape +func xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func atomicload64(ptr *uint64) uint64 + +//go:noescape +func atomicor8(ptr *uint8, val uint8) + +//go:noescape +func cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func atomicstore(ptr *uint32, val uint32) + +//go:noescape +func atomicstore64(ptr *uint64, val uint64) + +// atomicstorep cannot have a go:noescape annotation. +// See comment above for xchgp. + +//go:nosplit +func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { + atomicstorep1(noescape(ptr), new) +} + +func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/atomic_amd64x.c b/src/runtime/atomic_amd64x.c deleted file mode 100644 index 7be57ac95..000000000 --- a/src/runtime/atomic_amd64x.c +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build amd64 amd64p32 - -#include "runtime.h" -#include "textflag.h" - -#pragma textflag NOSPLIT -uint32 -runtime·atomicload(uint32 volatile* addr) -{ - return *addr; -} - -#pragma textflag NOSPLIT -uint64 -runtime·atomicload64(uint64 volatile* addr) -{ - return *addr; -} - -#pragma textflag NOSPLIT -void* -runtime·atomicloadp(void* volatile* addr) -{ - return *addr; -} diff --git a/src/runtime/atomic_amd64x.go b/src/runtime/atomic_amd64x.go new file mode 100644 index 000000000..f2dd58411 --- /dev/null +++ b/src/runtime/atomic_amd64x.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 amd64p32 + +package runtime + +import "unsafe" + +// The calls to nop are to keep these functions from being inlined. +// If they are inlined we have no guarantee that later rewrites of the +// code by optimizers will preserve the relative order of memory accesses. + +//go:nosplit +func atomicload(ptr *uint32) uint32 { + nop() + return *ptr +} + +//go:nosplit +func atomicloadp(ptr unsafe.Pointer) unsafe.Pointer { + nop() + return *(*unsafe.Pointer)(ptr) +} + +//go:nosplit +func atomicload64(ptr *uint64) uint64 { + nop() + return *ptr +} + +//go:noescape +func xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func xadd64(ptr *uint64, delta int64) uint64 + +//go:noescape +func xchg(ptr *uint32, new uint32) uint32 + +//go:noescape +func xchg64(ptr *uint64, new uint64) uint64 + +// xchgp cannot have a go:noescape annotation, because +// while ptr does not escape, new does. If new is marked as +// not escaping, the compiler will make incorrect escape analysis +// decisions about the value being xchg'ed. +// Instead, make xchgp a wrapper around the actual atomic. +// When calling the wrapper we mark ptr as noescape explicitly. + +//go:nosplit +func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { + return xchgp1(noescape(ptr), new) +} + +func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer + +//go:noescape +func xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func atomicor8(ptr *uint8, val uint8) + +//go:noescape +func cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func atomicstore(ptr *uint32, val uint32) + +//go:noescape +func atomicstore64(ptr *uint64, val uint64) + +// atomicstorep cannot have a go:noescape annotation. +// See comment above for xchgp. + +//go:nosplit +func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { + atomicstorep1(noescape(ptr), new) +} + +func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/atomic_arm.go b/src/runtime/atomic_arm.go index b1632cdd1..fd55a0aca 100644 --- a/src/runtime/atomic_arm.go +++ b/src/runtime/atomic_arm.go @@ -85,7 +85,7 @@ func atomicstore(addr *uint32, v uint32) { //go:nosplit func cas64(addr *uint64, old, new uint64) bool { var ok bool - onM(func() { + systemstack(func() { lock(addrLock(addr)) if *addr == old { *addr = new @@ -99,7 +99,7 @@ func cas64(addr *uint64, old, new uint64) bool { //go:nosplit func xadd64(addr *uint64, delta int64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr + uint64(delta) *addr = r @@ -111,7 +111,7 @@ func xadd64(addr *uint64, delta int64) uint64 { //go:nosplit func xchg64(addr *uint64, v uint64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr *addr = v @@ -123,7 +123,7 @@ func xchg64(addr *uint64, v uint64) uint64 { //go:nosplit func atomicload64(addr *uint64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr unlock(addrLock(addr)) @@ -133,7 +133,7 @@ func atomicload64(addr *uint64) uint64 { //go:nosplit func atomicstore64(addr *uint64, v uint64) { - onM(func() { + systemstack(func() { lock(addrLock(addr)) *addr = v unlock(addrLock(addr)) diff --git a/src/runtime/atomic.go b/src/runtime/atomic_power64x.go index 7e9d9b3aa..a0dcf514b 100644 --- a/src/runtime/atomic.go +++ b/src/runtime/atomic_power64x.go @@ -1,8 +1,8 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !arm +// +build power64 power64le package runtime @@ -20,8 +20,19 @@ func xchg(ptr *uint32, new uint32) uint32 //go:noescape func xchg64(ptr *uint64, new uint64) uint64 -//go:noescape -func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer +// xchgp cannot have a go:noescape annotation, because +// while ptr does not escape, new does. If new is marked as +// not escaping, the compiler will make incorrect escape analysis +// decisions about the value being xchg'ed. +// Instead, make xchgp a wrapper around the actual atomic. +// When calling the wrapper we mark ptr as noescape explicitly. + +//go:nosplit +func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { + return xchgp1(noescape(ptr), new) +} + +func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer //go:noescape func xchguintptr(ptr *uintptr, new uintptr) uintptr @@ -47,5 +58,12 @@ func atomicstore(ptr *uint32, val uint32) //go:noescape func atomicstore64(ptr *uint64, val uint64) -//go:noescape -func atomicstorep(ptr unsafe.Pointer, val unsafe.Pointer) +// atomicstorep cannot have a go:noescape annotation. +// See comment above for xchgp. + +//go:nosplit +func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { + atomicstorep1(noescape(ptr), new) +} + +func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/atomic_power64x.s b/src/runtime/atomic_power64x.s new file mode 100644 index 000000000..e72871761 --- /dev/null +++ b/src/runtime/atomic_power64x.s @@ -0,0 +1,40 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le + +#include "textflag.h" + +// uint32 runtime·atomicload(uint32 volatile* addr) +TEXT ·atomicload(SB),NOSPLIT,$-8-12 + MOVD 0(FP), R3 + SYNC + MOVWZ 0(R3), R3 + CMPW R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVW R3, ret+8(FP) + RETURN + +// uint64 runtime·atomicload64(uint64 volatile* addr) +TEXT ·atomicload64(SB),NOSPLIT,$-8-16 + MOVD 0(FP), R3 + SYNC + MOVD 0(R3), R3 + CMP R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVD R3, ret+8(FP) + RETURN + +// void *runtime·atomicloadp(void *volatile *addr) +TEXT ·atomicloadp(SB),NOSPLIT,$-8-16 + MOVD 0(FP), R3 + SYNC + MOVD 0(R3), R3 + CMP R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVD R3, ret+8(FP) + RETURN diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go new file mode 100644 index 000000000..7e6b253af --- /dev/null +++ b/src/runtime/cgo.go @@ -0,0 +1,23 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:cgo_export_static main + +// Filled in by runtime/cgo when linked into binary. + +//go:linkname _cgo_init _cgo_init +//go:linkname _cgo_malloc _cgo_malloc +//go:linkname _cgo_free _cgo_free +//go:linkname _cgo_thread_start _cgo_thread_start + +var ( + _cgo_init unsafe.Pointer + _cgo_malloc unsafe.Pointer + _cgo_free unsafe.Pointer + _cgo_thread_start unsafe.Pointer +) diff --git a/src/runtime/cgo/callbacks.c b/src/runtime/cgo/callbacks.c deleted file mode 100644 index 282beeea8..000000000 --- a/src/runtime/cgo/callbacks.c +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "../runtime.h" -#include "../cgocall.h" -#include "textflag.h" - -// These utility functions are available to be called from code -// compiled with gcc via crosscall2. - -// The declaration of crosscall2 is: -// void crosscall2(void (*fn)(void *, int), void *, int); -// -// We need to export the symbol crosscall2 in order to support -// callbacks from shared libraries. This applies regardless of -// linking mode. -#pragma cgo_export_static crosscall2 -#pragma cgo_export_dynamic crosscall2 - -// Allocate memory. This allocates the requested number of bytes in -// memory controlled by the Go runtime. The allocated memory will be -// zeroed. You are responsible for ensuring that the Go garbage -// collector can see a pointer to the allocated memory for as long as -// it is valid, e.g., by storing a pointer in a local variable in your -// C function, or in memory allocated by the Go runtime. If the only -// pointers are in a C global variable or in memory allocated via -// malloc, then the Go garbage collector may collect the memory. - -// Call like this in code compiled with gcc: -// struct { size_t len; void *ret; } a; -// a.len = /* number of bytes to allocate */; -// crosscall2(_cgo_allocate, &a, sizeof a); -// /* Here a.ret is a pointer to the allocated memory. */ - -void runtime·_cgo_allocate_internal(void); - -#pragma cgo_export_static _cgo_allocate -#pragma cgo_export_dynamic _cgo_allocate -#pragma textflag NOSPLIT -void -_cgo_allocate(void *a, int32 n) -{ - runtime·cgocallback((void(*)(void))runtime·_cgo_allocate_internal, a, n); -} - -// Panic. The argument is converted into a Go string. - -// Call like this in code compiled with gcc: -// struct { const char *p; } a; -// a.p = /* string to pass to panic */; -// crosscall2(_cgo_panic, &a, sizeof a); -// /* The function call will not return. */ - -void runtime·_cgo_panic_internal(void); - -#pragma cgo_export_static _cgo_panic -#pragma cgo_export_dynamic _cgo_panic -#pragma textflag NOSPLIT -void -_cgo_panic(void *a, int32 n) -{ - runtime·cgocallback((void(*)(void))runtime·_cgo_panic_internal, a, n); -} - -#pragma cgo_import_static x_cgo_init -extern void x_cgo_init(G*); -void (*_cgo_init)(G*) = x_cgo_init; - -#pragma cgo_import_static x_cgo_malloc -extern void x_cgo_malloc(void*); -void (*_cgo_malloc)(void*) = x_cgo_malloc; - -#pragma cgo_import_static x_cgo_free -extern void x_cgo_free(void*); -void (*_cgo_free)(void*) = x_cgo_free; - -#pragma cgo_import_static x_cgo_thread_start -extern void x_cgo_thread_start(void*); -void (*_cgo_thread_start)(void*) = x_cgo_thread_start; - -#pragma cgo_export_static _cgo_topofstack -#pragma cgo_export_dynamic _cgo_topofstack diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go new file mode 100644 index 000000000..1e8b59054 --- /dev/null +++ b/src/runtime/cgo/callbacks.go @@ -0,0 +1,95 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +import "unsafe" + +// These utility functions are available to be called from code +// compiled with gcc via crosscall2. + +// cgocallback is defined in runtime +//go:linkname _runtime_cgocallback runtime.cgocallback +func _runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr) + +// The declaration of crosscall2 is: +// void crosscall2(void (*fn)(void *, int), void *, int); +// +// We need to export the symbol crosscall2 in order to support +// callbacks from shared libraries. This applies regardless of +// linking mode. +//go:cgo_export_static crosscall2 +//go:cgo_export_dynamic crosscall2 + +// Allocate memory. This allocates the requested number of bytes in +// memory controlled by the Go runtime. The allocated memory will be +// zeroed. You are responsible for ensuring that the Go garbage +// collector can see a pointer to the allocated memory for as long as +// it is valid, e.g., by storing a pointer in a local variable in your +// C function, or in memory allocated by the Go runtime. If the only +// pointers are in a C global variable or in memory allocated via +// malloc, then the Go garbage collector may collect the memory. + +// Call like this in code compiled with gcc: +// struct { size_t len; void *ret; } a; +// a.len = /* number of bytes to allocate */; +// crosscall2(_cgo_allocate, &a, sizeof a); +// /* Here a.ret is a pointer to the allocated memory. */ + +//go:linkname _runtime_cgo_allocate_internal runtime._cgo_allocate_internal +var _runtime_cgo_allocate_internal byte + +//go:linkname _cgo_allocate _cgo_allocate +//go:cgo_export_static _cgo_allocate +//go:cgo_export_dynamic _cgo_allocate +//go:nosplit +func _cgo_allocate(a unsafe.Pointer, n int32) { + _runtime_cgocallback(unsafe.Pointer(&_runtime_cgo_allocate_internal), a, uintptr(n)) +} + +// Panic. The argument is converted into a Go string. + +// Call like this in code compiled with gcc: +// struct { const char *p; } a; +// a.p = /* string to pass to panic */; +// crosscall2(_cgo_panic, &a, sizeof a); +// /* The function call will not return. */ + +//go:linkname _runtime_cgo_panic_internal runtime._cgo_panic_internal +var _runtime_cgo_panic_internal byte + +//go:linkname _cgo_panic _cgo_panic +//go:cgo_export_static _cgo_panic +//go:cgo_export_dynamic _cgo_panic +//go:nosplit +func _cgo_panic(a unsafe.Pointer, n int32) { + _runtime_cgocallback(unsafe.Pointer(&_runtime_cgo_panic_internal), a, uintptr(n)) +} + +//go:cgo_import_static x_cgo_init +//go:linkname x_cgo_init x_cgo_init +//go:linkname _cgo_init _cgo_init +var x_cgo_init byte +var _cgo_init = &x_cgo_init + +//go:cgo_import_static x_cgo_malloc +//go:linkname x_cgo_malloc x_cgo_malloc +//go:linkname _cgo_malloc _cgo_malloc +var x_cgo_malloc byte +var _cgo_malloc = &x_cgo_malloc + +//go:cgo_import_static x_cgo_free +//go:linkname x_cgo_free x_cgo_free +//go:linkname _cgo_free _cgo_free +var x_cgo_free byte +var _cgo_free = &x_cgo_free + +//go:cgo_import_static x_cgo_thread_start +//go:linkname x_cgo_thread_start x_cgo_thread_start +//go:linkname _cgo_thread_start _cgo_thread_start +var x_cgo_thread_start byte +var _cgo_thread_start = &x_cgo_thread_start + +//go:cgo_export_static _cgo_topofstack +//go:cgo_export_dynamic _cgo_topofstack diff --git a/src/runtime/cgo/dragonfly.c b/src/runtime/cgo/dragonfly.go index c233c8ba9..69d52b5b2 100644 --- a/src/runtime/cgo/dragonfly.c +++ b/src/runtime/cgo/dragonfly.go @@ -4,16 +4,16 @@ // +build dragonfly -#include "textflag.h" +package cgo + +import _ "unsafe" // for go:linkname // Supply environ and __progname, because we don't // link against the standard DragonFly crt0.o and the // libc dynamic library needs them. -#pragma dataflag NOPTR -char *environ[1]; -#pragma dataflag NOPTR -char *__progname; +//go:linkname _environ environ +//go:linkname _progname __progname -#pragma dynexport environ environ -#pragma dynexport __progname __progname +var _environ uintptr +var _progname uintptr diff --git a/src/runtime/cgo/freebsd.c b/src/runtime/cgo/freebsd.go index 4876b2abe..99cf3fbca 100644 --- a/src/runtime/cgo/freebsd.c +++ b/src/runtime/cgo/freebsd.go @@ -4,16 +4,19 @@ // +build freebsd -#include "textflag.h" +package cgo + +import _ "unsafe" // for go:linkname // Supply environ and __progname, because we don't // link against the standard FreeBSD crt0.o and the // libc dynamic library needs them. -#pragma dataflag NOPTR -char *environ[1]; -#pragma dataflag NOPTR -char *__progname; +//go:linkname _environ environ +//go:linkname _progname __progname + +//go:cgo_export_dynamic environ +//go:cgo_export_dynamic __progname -#pragma dynexport environ environ -#pragma dynexport __progname __progname +var _environ uintptr +var _progname uintptr diff --git a/src/runtime/cgo/iscgo.c b/src/runtime/cgo/iscgo.go index 0907a1958..61cba73d2 100644 --- a/src/runtime/cgo/iscgo.c +++ b/src/runtime/cgo/iscgo.go @@ -9,7 +9,12 @@ // correctly, and sometimes they break. This variable is a // backup: it depends only on old C style static linking rules. -#include "../runtime.h" +package cgo -bool runtime·iscgo = 1; -uint32 runtime·needextram = 1; // create an extra M on first cgo call +import _ "unsafe" // for go:linkname + +//go:linkname _iscgo runtime.iscgo +var _iscgo bool = true + +//go:linkname _needextram runtime.needextram +var _needextram uint32 = 1 // create an extra M on first cgo call diff --git a/src/runtime/cgo/netbsd.c b/src/runtime/cgo/netbsd.go index 076cc87f1..ac6b18a93 100644 --- a/src/runtime/cgo/netbsd.c +++ b/src/runtime/cgo/netbsd.go @@ -4,16 +4,16 @@ // +build netbsd -#include "textflag.h" +package cgo + +import _ "unsafe" // for go:linkname // Supply environ and __progname, because we don't // link against the standard NetBSD crt0.o and the // libc dynamic library needs them. -#pragma dataflag NOPTR -char *environ[1]; -#pragma dataflag NOPTR -char *__progname; +//go:linkname _environ environ +//go:linkname _progname __progname -#pragma dynexport environ environ -#pragma dynexport __progname __progname +var _environ uintptr +var _progname uintptr diff --git a/src/runtime/cgo/openbsd.c b/src/runtime/cgo/openbsd.go index 476649544..61af3a8e7 100644 --- a/src/runtime/cgo/openbsd.c +++ b/src/runtime/cgo/openbsd.go @@ -4,24 +4,28 @@ // +build openbsd -#include "textflag.h" +package cgo + +import _ "unsafe" // for go:linkname // Supply environ, __progname and __guard_local, because // we don't link against the standard OpenBSD crt0.o and // the libc dynamic library needs them. -#pragma dataflag NOPTR -char *environ[1]; -#pragma dataflag NOPTR -char *__progname; -long __guard_local; +//go:linkname _environ environ +//go:linkname _progname __progname +//go:linkname _guard_local __guard_local + +var _environ uintptr +var _progname uintptr +var _guard_local uintptr -#pragma dynexport environ environ -#pragma dynexport __progname __progname +//go:cgo_export_dynamic environ environ +//go:cgo_export_dynamic __progname __progname // This is normally marked as hidden and placed in the // .openbsd.randomdata section. -#pragma dynexport __guard_local __guard_local +//go:cgo_export_dynamic __guard_local __guard_local // We override pthread_create to support PT_TLS. -#pragma dynexport pthread_create pthread_create +//go:cgo_export_dynamic pthread_create pthread_create diff --git a/src/runtime/cgo/setenv.c b/src/runtime/cgo/setenv.c deleted file mode 100644 index 76d88cbf1..000000000 --- a/src/runtime/cgo/setenv.c +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux netbsd openbsd - -#pragma cgo_import_static x_cgo_setenv -#pragma cgo_import_static x_cgo_unsetenv - -void x_cgo_setenv(char**); -void (*runtime·_cgo_setenv)(char**) = x_cgo_setenv; -void x_cgo_unsetenv(char**); -void (*runtime·_cgo_unsetenv)(char**) = x_cgo_unsetenv; diff --git a/src/runtime/cgo/setenv.go b/src/runtime/cgo/setenv.go new file mode 100644 index 000000000..97c8c6ac9 --- /dev/null +++ b/src/runtime/cgo/setenv.go @@ -0,0 +1,21 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package cgo + +import _ "unsafe" // for go:linkname + +//go:cgo_import_static x_cgo_setenv +//go:linkname x_cgo_setenv x_cgo_setenv +//go:linkname _cgo_setenv runtime._cgo_setenv +var x_cgo_setenv byte +var _cgo_setenv = &x_cgo_setenv + +//go:cgo_import_static x_cgo_unsetenv +//go:linkname x_cgo_unsetenv x_cgo_unsetenv +//go:linkname _cgo_unsetenv runtime._cgo_unsetenv +var x_cgo_unsetenv byte +var _cgo_unsetenv = &x_cgo_unsetenv diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 7fd91469e..258cabfc8 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -103,7 +103,7 @@ func cgocall_errno(fn, arg unsafe.Pointer) int32 { // Create an extra M for callbacks on threads not created by Go on first cgo call. if needextram == 1 && cas(&needextram, 1, 0) { - onM(newextram) + systemstack(newextram) } /* @@ -127,9 +127,9 @@ func cgocall_errno(fn, arg unsafe.Pointer) int32 { * so it is safe to call while "in a system call", outside * the $GOMAXPROCS accounting. */ - entersyscall() + entersyscall(0) errno := asmcgocall_errno(fn, arg) - exitsyscall() + exitsyscall(0) return errno } @@ -153,17 +153,13 @@ func endcgo(mp *m) { // Helper functions for cgo code. -// Filled by schedinit from corresponding C variables, -// which are in turn filled in by dynamic linker when Cgo is available. -var cgoMalloc, cgoFree unsafe.Pointer - func cmalloc(n uintptr) unsafe.Pointer { var args struct { n uint64 ret unsafe.Pointer } args.n = uint64(n) - cgocall(cgoMalloc, unsafe.Pointer(&args)) + cgocall(_cgo_malloc, unsafe.Pointer(&args)) if args.ret == nil { gothrow("C malloc failed") } @@ -171,7 +167,7 @@ func cmalloc(n uintptr) unsafe.Pointer { } func cfree(p unsafe.Pointer) { - cgocall(cgoFree, p) + cgocall(_cgo_free, p) } // Call from C back to Go. @@ -189,17 +185,17 @@ func cgocallbackg() { // save syscall* and let reentersyscall restore them. savedsp := unsafe.Pointer(gp.syscallsp) savedpc := gp.syscallpc - exitsyscall() // coming out of cgo call + exitsyscall(0) // coming out of cgo call cgocallbackg1() // going back to cgo call - reentersyscall(savedpc, savedsp) + reentersyscall(savedpc, uintptr(savedsp)) } func cgocallbackg1() { gp := getg() if gp.m.needextram { gp.m.needextram = false - onM(newextram) + systemstack(newextram) } // Add entry to defer stack in case of panic. diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 0eb87df74..330422ad0 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -26,7 +26,7 @@ func makechan(t *chantype, size int64) *hchan { if hchanSize%maxAlign != 0 || elem.align > maxAlign { gothrow("makechan: bad alignment") } - if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (maxmem-hchanSize)/uintptr(elem.size)) { + if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/uintptr(elem.size)) { panic("makechan: size out of range") } diff --git a/src/runtime/chan.h b/src/runtime/chan.h deleted file mode 100644 index c34ff1533..000000000 --- a/src/runtime/chan.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define MAXALIGN 8 - -typedef struct WaitQ WaitQ; -typedef struct Select Select; -typedef struct Scase Scase; - -struct WaitQ -{ - SudoG* first; - SudoG* last; -}; - -struct Hchan -{ - uintgo qcount; // total data in the q - uintgo dataqsiz; // size of the circular q - byte* buf; - uint16 elemsize; - uint32 closed; - Type* elemtype; // element type - uintgo sendx; // send index - uintgo recvx; // receive index - WaitQ recvq; // list of recv waiters - WaitQ sendq; // list of send waiters - Mutex lock; -}; - -// Buffer follows Hchan immediately in memory. -// chanbuf(c, i) is pointer to the i'th slot in the buffer. -#define chanbuf(c, i) ((byte*)((c)->buf)+(uintptr)(c)->elemsize*(i)) - -enum -{ - debug = 0, - - // Scase.kind - CaseRecv, - CaseSend, - CaseDefault, -}; - -// Known to compiler. -// Changes here must also be made in src/cmd/gc/select.c's selecttype. -struct Scase -{ - void* elem; // data element - Hchan* chan; // chan - uintptr pc; // return pc - uint16 kind; - uint16 so; // vararg of selected bool - bool* receivedp; // pointer to received bool (recv2) - int64 releasetime; -}; - -// Known to compiler. -// Changes here must also be made in src/cmd/gc/select.c's selecttype. -struct Select -{ - uint16 tcase; // total count of scase[] - uint16 ncase; // currently filled scase[] - uint16* pollorder; // case poll order - Hchan** lockorder; // channel lock order - Scase scase[1]; // one per case (in order of appearance) -}; diff --git a/src/runtime/chan1.go b/src/runtime/chan1.go new file mode 100644 index 000000000..000775b1e --- /dev/null +++ b/src/runtime/chan1.go @@ -0,0 +1,61 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//#define MAXALIGN 8 + +type waitq struct { + first *sudog + last *sudog +} + +type hchan struct { + qcount uint // total data in the q + dataqsiz uint // size of the circular q + buf *byte + elemsize uint16 + closed uint32 + elemtype *_type // element type + sendx uint // send index + recvx uint // receive index + recvq waitq // list of recv waiters + sendq waitq // list of send waiters + lock mutex +} + +// Buffer follows Hchan immediately in memory. +// chanbuf(c, i) is pointer to the i'th slot in the buffer. +// #define chanbuf(c, i) ((byte*)((c)->buf)+(uintptr)(c)->elemsize*(i)) + +const ( + // scase.kind + _CaseRecv = iota + _CaseSend + _CaseDefault +) + +// Known to compiler. +// Changes here must also be made in src/cmd/gc/select.c's selecttype. +type scase struct { + elem unsafe.Pointer // data element + _chan *hchan // chan + pc uintptr // return pc + kind uint16 + so uint16 // vararg of selected bool + receivedp *bool // pointer to received bool (recv2) + releasetime int64 +} + +// Known to compiler. +// Changes here must also be made in src/cmd/gc/select.c's selecttype. +type _select struct { + tcase uint16 // total count of scase[] + ncase uint16 // currently filled scase[] + pollorder *uint16 // case poll order + lockorder **hchan // channel lock order + scase [1]scase // one per case (in order of appearance) +} diff --git a/src/runtime/complex.go b/src/runtime/complex.go index ec50f8947..73f1161a5 100644 --- a/src/runtime/complex.go +++ b/src/runtime/complex.go @@ -4,28 +4,47 @@ package runtime +func isposinf(f float64) bool { return f > maxFloat64 } +func isneginf(f float64) bool { return f < -maxFloat64 } +func isnan(f float64) bool { return f != f } + +func nan() float64 { + var f float64 = 0 + return f / f +} + +func posinf() float64 { + var f float64 = maxFloat64 + return f * f +} + +func neginf() float64 { + var f float64 = maxFloat64 + return -f * f +} + func complex128div(n complex128, d complex128) complex128 { // Special cases as in C99. - ninf := real(n) == posinf || real(n) == neginf || - imag(n) == posinf || imag(n) == neginf - dinf := real(d) == posinf || real(d) == neginf || - imag(d) == posinf || imag(d) == neginf + ninf := isposinf(real(n)) || isneginf(real(n)) || + isposinf(imag(n)) || isneginf(imag(n)) + dinf := isposinf(real(d)) || isneginf(real(d)) || + isposinf(imag(d)) || isneginf(imag(d)) - nnan := !ninf && (real(n) != real(n) || imag(n) != imag(n)) - dnan := !dinf && (real(d) != real(d) || imag(d) != imag(d)) + nnan := !ninf && (isnan(real(n)) || isnan(imag(n))) + dnan := !dinf && (isnan(real(d)) || isnan(imag(d))) switch { case nnan || dnan: - return complex(nan, nan) + return complex(nan(), nan()) case ninf && !dinf: - return complex(posinf, posinf) + return complex(posinf(), posinf()) case !ninf && dinf: return complex(0, 0) case real(d) == 0 && imag(d) == 0: if real(n) == 0 && imag(n) == 0 { - return complex(nan, nan) + return complex(nan(), nan()) } else { - return complex(posinf, posinf) + return complex(posinf(), posinf()) } default: // Standard complex arithmetic, factored to avoid unnecessary overflow. diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index 8b1c1c632..d56678e21 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -101,12 +101,10 @@ var ( eod = [3]uintptr{0, 1, 0} ) -func setcpuprofilerate_m() // proc.c - func setcpuprofilerate(hz int32) { - g := getg() - g.m.scalararg[0] = uintptr(hz) - onM(setcpuprofilerate_m) + systemstack(func() { + setcpuprofilerate_m(hz) + }) } // lostProfileData is a no-op function used in profiles diff --git a/src/runtime/cputicks.go b/src/runtime/cputicks.go new file mode 100644 index 000000000..e0593d56e --- /dev/null +++ b/src/runtime/cputicks.go @@ -0,0 +1,11 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !arm + +package runtime + +// careful: cputicks is not guaranteed to be monotonic! In particular, we have +// noticed drift between cpus on certain os/arch combinations. See issue 8976. +func cputicks() int64 diff --git a/src/runtime/debug.go b/src/runtime/debug.go index 4414dd55d..105b79cfe 100644 --- a/src/runtime/debug.go +++ b/src/runtime/debug.go @@ -6,18 +6,6 @@ package runtime import "unsafe" -// Breakpoint executes a breakpoint trap. -func Breakpoint() - -// LockOSThread wires the calling goroutine to its current operating system thread. -// Until the calling goroutine exits or calls UnlockOSThread, it will always -// execute in that thread, and no other goroutine can. -func LockOSThread() - -// UnlockOSThread unwires the calling goroutine from its fixed operating system thread. -// If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op. -func UnlockOSThread() - // GOMAXPROCS sets the maximum number of CPUs that can be executing // simultaneously and returns the previous setting. If n < 1, it does not // change the current setting. @@ -37,14 +25,14 @@ func GOMAXPROCS(n int) int { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) // newprocs will be processed by starttheworld newprocs = int32(n) gp.m.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) return ret } @@ -66,5 +54,3 @@ func NumCgoCall() int64 { func NumGoroutine() int { return int(gcount()) } - -func gcount() int32 diff --git a/src/runtime/debug/stubs.s b/src/runtime/debug/stubs.s index d56274f2d..1e883b72c 100644 --- a/src/runtime/debug/stubs.s +++ b/src/runtime/debug/stubs.s @@ -7,6 +7,12 @@ #ifdef GOARCH_arm #define JMP B #endif +#ifdef GOARCH_power64 +#define JMP BR +#endif +#ifdef GOARCH_power64le +#define JMP BR +#endif TEXT ·setMaxStack(SB),NOSPLIT,$0-0 JMP runtime·setMaxStack(SB) diff --git a/src/runtime/defs.c b/src/runtime/defs.c deleted file mode 100644 index b0a9b20d7..000000000 --- a/src/runtime/defs.c +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file is compiled by cmd/dist to obtain debug information -// about the given header files. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "type.h" -#include "race.h" -#include "chan.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" diff --git a/src/runtime/defs1_linux.go b/src/runtime/defs1_linux.go index 392cc4ab5..87c6e02a4 100644 --- a/src/runtime/defs1_linux.go +++ b/src/runtime/defs1_linux.go @@ -15,12 +15,14 @@ package runtime /* #include <ucontext.h> #include <fcntl.h> +#include <asm/signal.h> */ import "C" const ( - O_RDONLY = C.O_RDONLY - O_CLOEXEC = C.O_CLOEXEC + O_RDONLY = C.O_RDONLY + O_CLOEXEC = C.O_CLOEXEC + SA_RESTORER = C.SA_RESTORER ) type Usigset C.__sigset_t diff --git a/src/runtime/defs1_netbsd_386.go b/src/runtime/defs1_netbsd_386.go new file mode 100644 index 000000000..e39fd04c7 --- /dev/null +++ b/src/runtime/defs1_netbsd_386.go @@ -0,0 +1,171 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_386.go + +package runtime + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x6 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0 + _EV_ERROR = 0x4000 + _EVFILT_READ = 0x0 + _EVFILT_WRITE = 0x1 +) + +type sigaltstackt struct { + ss_sp *byte + ss_size uint32 + ss_flags int32 +} + +type sigset struct { + __bits [4]uint32 +} + +type siginfo struct { + _signo int32 + _code int32 + _errno int32 + _reason [20]byte +} + +type stackt struct { + ss_sp *byte + ss_size uint32 + ss_flags int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +type timeval struct { + tv_sec int64 + tv_usec int32 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type mcontextt struct { + __gregs [19]int32 + __fpregs [644]byte + _mc_tlsbase int32 +} + +type ucontextt struct { + uc_flags uint32 + uc_link *ucontextt + uc_sigmask sigset + uc_stack stackt + uc_mcontext mcontextt + __uc_pad [4]int32 +} + +type keventt struct { + ident uint32 + filter uint32 + flags uint32 + fflags uint32 + data int64 + udata *byte +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_386.go + +const ( + _REG_GS = 0x0 + _REG_FS = 0x1 + _REG_ES = 0x2 + _REG_DS = 0x3 + _REG_EDI = 0x4 + _REG_ESI = 0x5 + _REG_EBP = 0x6 + _REG_ESP = 0x7 + _REG_EBX = 0x8 + _REG_EDX = 0x9 + _REG_ECX = 0xa + _REG_EAX = 0xb + _REG_TRAPNO = 0xc + _REG_ERR = 0xd + _REG_EIP = 0xe + _REG_CS = 0xf + _REG_EFL = 0x10 + _REG_UESP = 0x11 + _REG_SS = 0x12 +) diff --git a/src/runtime/defs1_netbsd_amd64.go b/src/runtime/defs1_netbsd_amd64.go new file mode 100644 index 000000000..cca701e5b --- /dev/null +++ b/src/runtime/defs1_netbsd_amd64.go @@ -0,0 +1,183 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go + +package runtime + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x6 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0 + _EV_ERROR = 0x4000 + _EVFILT_READ = 0x0 + _EVFILT_WRITE = 0x1 +) + +type sigaltstackt struct { + ss_sp *byte + ss_size uint64 + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigset struct { + __bits [4]uint32 +} + +type siginfo struct { + _signo int32 + _code int32 + _errno int32 + _pad int32 + _reason [24]byte +} + +type stackt struct { + ss_sp *byte + ss_size uint64 + ss_flags int32 + pad_cgo_0 [4]byte +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +type timeval struct { + tv_sec int64 + tv_usec int32 + pad_cgo_0 [4]byte +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type mcontextt struct { + __gregs [26]uint64 + _mc_tlsbase uint64 + __fpregs [512]int8 +} + +type ucontextt struct { + uc_flags uint32 + pad_cgo_0 [4]byte + uc_link *ucontextt + uc_sigmask sigset + uc_stack stackt + uc_mcontext mcontextt +} + +type keventt struct { + ident uint64 + filter uint32 + flags uint32 + fflags uint32 + pad_cgo_0 [4]byte + data int64 + udata *byte +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go + +const ( + _REG_RDI = 0x0 + _REG_RSI = 0x1 + _REG_RDX = 0x2 + _REG_RCX = 0x3 + _REG_R8 = 0x4 + _REG_R9 = 0x5 + _REG_R10 = 0x6 + _REG_R11 = 0x7 + _REG_R12 = 0x8 + _REG_R13 = 0x9 + _REG_R14 = 0xa + _REG_R15 = 0xb + _REG_RBP = 0xc + _REG_RBX = 0xd + _REG_RAX = 0xe + _REG_GS = 0xf + _REG_FS = 0x10 + _REG_ES = 0x11 + _REG_DS = 0x12 + _REG_TRAPNO = 0x13 + _REG_ERR = 0x14 + _REG_RIP = 0x15 + _REG_CS = 0x16 + _REG_RFLAGS = 0x17 + _REG_RSP = 0x18 + _REG_SS = 0x19 +) diff --git a/src/runtime/defs1_netbsd_arm.go b/src/runtime/defs1_netbsd_arm.go new file mode 100644 index 000000000..54ddf3882 --- /dev/null +++ b/src/runtime/defs1_netbsd_arm.go @@ -0,0 +1,170 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go + +package runtime + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x6 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0 + _EV_ERROR = 0x4000 + _EVFILT_READ = 0x0 + _EVFILT_WRITE = 0x1 +) + +type sigaltstackt struct { + ss_sp *byte + ss_size uint32 + ss_flags int32 +} + +type sigset struct { + __bits [4]uint32 +} + +type siginfo struct { + _signo int32 + _code int32 + _errno int32 + _reason [20]byte +} + +type stackt struct { + ss_sp *byte + ss_size uint32 + ss_flags int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +type timeval struct { + tv_sec int64 + tv_usec int32 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type mcontextt struct { + __gregs [17]uint32 + __fpu [4 + 8*32 + 4]byte // EABI + // __fpu [4+4*33+4]byte // not EABI + _mc_tlsbase uint32 +} + +type ucontextt struct { + uc_flags uint32 + uc_link *ucontextt + uc_sigmask sigset + uc_stack stackt + uc_mcontext mcontextt + __uc_pad [2]int32 +} + +type keventt struct { + ident uint32 + filter uint32 + flags uint32 + fflags uint32 + data int64 + udata *byte +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go + +const ( + _REG_R0 = 0x0 + _REG_R1 = 0x1 + _REG_R2 = 0x2 + _REG_R3 = 0x3 + _REG_R4 = 0x4 + _REG_R5 = 0x5 + _REG_R6 = 0x6 + _REG_R7 = 0x7 + _REG_R8 = 0x8 + _REG_R9 = 0x9 + _REG_R10 = 0xa + _REG_R11 = 0xb + _REG_R12 = 0xc + _REG_R13 = 0xd + _REG_R14 = 0xe + _REG_R15 = 0xf + _REG_CPSR = 0x10 +) diff --git a/src/runtime/defs1_solaris_amd64.go b/src/runtime/defs1_solaris_amd64.go new file mode 100644 index 000000000..3bb6f69bf --- /dev/null +++ b/src/runtime/defs1_solaris_amd64.go @@ -0,0 +1,245 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_solaris.go defs_solaris_amd64.go + +package runtime + +const ( + _EINTR = 0x4 + _EBADF = 0x9 + _EFAULT = 0xe + _EAGAIN = 0xb + _ETIMEDOUT = 0x91 + _EWOULDBLOCK = 0xb + _EINPROGRESS = 0x96 + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x100 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x8 + _SA_RESTART = 0x4 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x15 + _SIGSTOP = 0x17 + _SIGTSTP = 0x18 + _SIGCONT = 0x19 + _SIGCHLD = 0x12 + _SIGTTIN = 0x1a + _SIGTTOU = 0x1b + _SIGIO = 0x16 + _SIGXCPU = 0x1e + _SIGXFSZ = 0x1f + _SIGVTALRM = 0x1c + _SIGPROF = 0x1d + _SIGWINCH = 0x14 + _SIGUSR1 = 0x10 + _SIGUSR2 = 0x11 + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + __SC_NPROCESSORS_ONLN = 0xf + + _PTHREAD_CREATE_DETACHED = 0x40 + + _FORK_NOSIGCHLD = 0x1 + _FORK_WAITPID = 0x2 + + _MAXHOSTNAMELEN = 0x100 + + _O_NONBLOCK = 0x80 + _FD_CLOEXEC = 0x1 + _F_GETFL = 0x3 + _F_SETFL = 0x4 + _F_SETFD = 0x2 + + _POLLIN = 0x1 + _POLLOUT = 0x4 + _POLLHUP = 0x10 + _POLLERR = 0x8 + + _PORT_SOURCE_FD = 0x4 +) + +type semt struct { + sem_count uint32 + sem_type uint16 + sem_magic uint16 + sem_pad1 [3]uint64 + sem_pad2 [2]uint64 +} + +type sigaltstackt struct { + ss_sp *byte + ss_size uint64 + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigset struct { + __sigbits [4]uint32 +} + +type stackt struct { + ss_sp *byte + ss_size uint64 + ss_flags int32 + pad_cgo_0 [4]byte +} + +type siginfo struct { + si_signo int32 + si_code int32 + si_errno int32 + si_pad int32 + __data [240]byte +} + +type sigactiont struct { + sa_flags int32 + pad_cgo_0 [4]byte + _funcptr [8]byte + sa_mask sigset +} + +type fpregset struct { + fp_reg_set [528]byte +} + +type mcontext struct { + gregs [28]int64 + fpregs fpregset +} + +type ucontext struct { + uc_flags uint64 + uc_link *ucontext + uc_sigmask sigset + uc_stack stackt + pad_cgo_0 [8]byte + uc_mcontext mcontext + uc_filler [5]int64 + pad_cgo_1 [8]byte +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type portevent struct { + portev_events int32 + portev_source uint16 + portev_pad uint16 + portev_object uint64 + portev_user *byte +} + +type pthread uint32 +type pthreadattr struct { + __pthread_attrp *byte +} + +type stat struct { + st_dev uint64 + st_ino uint64 + st_mode uint32 + st_nlink uint32 + st_uid uint32 + st_gid uint32 + st_rdev uint64 + st_size int64 + st_atim timespec + st_mtim timespec + st_ctim timespec + st_blksize int32 + pad_cgo_0 [4]byte + st_blocks int64 + st_fstype [16]int8 +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_solaris.go defs_solaris_amd64.go + +const ( + _REG_RDI = 0x8 + _REG_RSI = 0x9 + _REG_RDX = 0xc + _REG_RCX = 0xd + _REG_R8 = 0x7 + _REG_R9 = 0x6 + _REG_R10 = 0x5 + _REG_R11 = 0x4 + _REG_R12 = 0x3 + _REG_R13 = 0x2 + _REG_R14 = 0x1 + _REG_R15 = 0x0 + _REG_RBP = 0xa + _REG_RBX = 0xb + _REG_RAX = 0xe + _REG_GS = 0x17 + _REG_FS = 0x16 + _REG_ES = 0x18 + _REG_DS = 0x19 + _REG_TRAPNO = 0xf + _REG_ERR = 0x10 + _REG_RIP = 0x11 + _REG_CS = 0x12 + _REG_RFLAGS = 0x13 + _REG_RSP = 0x14 + _REG_SS = 0x15 +) diff --git a/src/runtime/defs3_linux.go b/src/runtime/defs3_linux.go new file mode 100644 index 000000000..3551a4fa9 --- /dev/null +++ b/src/runtime/defs3_linux.go @@ -0,0 +1,43 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -cdefs + +GOARCH=power64 cgo -cdefs defs_linux.go defs3_linux.go > defs_linux_power64.h +*/ + +package runtime + +/* +#define size_t __kernel_size_t +#define sigset_t __sigset_t // rename the sigset_t here otherwise cgo will complain about "inconsistent definitions for C.sigset_t" +#define _SYS_TYPES_H // avoid inclusion of sys/types.h +#include <asm/ucontext.h> +#include <asm-generic/fcntl.h> +*/ +import "C" + +const ( + O_RDONLY = C.O_RDONLY + O_CLOEXEC = C.O_CLOEXEC + SA_RESTORER = 0 // unused +) + +type Usigset C.__sigset_t + +// types used in sigcontext +type Ptregs C.struct_pt_regs +type Gregset C.elf_gregset_t +type FPregset C.elf_fpregset_t +type Vreg C.elf_vrreg_t + +type SigaltstackT C.struct_sigaltstack + +// PPC64 uses sigcontext in place of mcontext in ucontext. +// see http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/arch/powerpc/include/uapi/asm/ucontext.h +type Sigcontext C.struct_sigcontext +type Ucontext C.struct_ucontext diff --git a/src/runtime/defs_android_arm.h b/src/runtime/defs_android_arm.h deleted file mode 100644 index 3611b3a10..000000000 --- a/src/runtime/defs_android_arm.h +++ /dev/null @@ -1,3 +0,0 @@ -// TODO: Generate using cgo like defs_linux_{386,amd64}.h - -#include "defs_linux_arm.h" diff --git a/src/runtime/defs_darwin_386.go b/src/runtime/defs_darwin_386.go new file mode 100644 index 000000000..cf4812f9f --- /dev/null +++ b/src/runtime/defs_darwin_386.go @@ -0,0 +1,382 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_darwin.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x5 + + _MACH_MSG_TYPE_MOVE_RECEIVE = 0x10 + _MACH_MSG_TYPE_MOVE_SEND = 0x11 + _MACH_MSG_TYPE_MOVE_SEND_ONCE = 0x12 + _MACH_MSG_TYPE_COPY_SEND = 0x13 + _MACH_MSG_TYPE_MAKE_SEND = 0x14 + _MACH_MSG_TYPE_MAKE_SEND_ONCE = 0x15 + _MACH_MSG_TYPE_COPY_RECEIVE = 0x16 + + _MACH_MSG_PORT_DESCRIPTOR = 0x0 + _MACH_MSG_OOL_DESCRIPTOR = 0x1 + _MACH_MSG_OOL_PORTS_DESCRIPTOR = 0x2 + _MACH_MSG_OOL_VOLATILE_DESCRIPTOR = 0x3 + + _MACH_MSGH_BITS_COMPLEX = 0x80000000 + + _MACH_SEND_MSG = 0x1 + _MACH_RCV_MSG = 0x2 + _MACH_RCV_LARGE = 0x4 + + _MACH_SEND_TIMEOUT = 0x10 + _MACH_SEND_INTERRUPT = 0x40 + _MACH_SEND_ALWAYS = 0x10000 + _MACH_SEND_TRAILER = 0x20000 + _MACH_RCV_TIMEOUT = 0x100 + _MACH_RCV_NOTIFY = 0x200 + _MACH_RCV_INTERRUPT = 0x400 + _MACH_RCV_OVERWRITE = 0x1000 + + _NDR_PROTOCOL_2_0 = 0x0 + _NDR_INT_BIG_ENDIAN = 0x0 + _NDR_INT_LITTLE_ENDIAN = 0x1 + _NDR_FLOAT_IEEE = 0x0 + _NDR_CHAR_ASCII = 0x0 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + _SA_USERTRAMP = 0x100 + _SA_64REGSET = 0x200 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x7 + _FPE_INTOVF = 0x8 + _FPE_FLTDIV = 0x1 + _FPE_FLTOVF = 0x2 + _FPE_FLTUND = 0x3 + _FPE_FLTRES = 0x4 + _FPE_FLTINV = 0x5 + _FPE_FLTSUB = 0x6 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0x40 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type machbody struct { + msgh_descriptor_count uint32 +} + +type machheader struct { + msgh_bits uint32 + msgh_size uint32 + msgh_remote_port uint32 + msgh_local_port uint32 + msgh_reserved uint32 + msgh_id int32 +} + +type machndr struct { + mig_vers uint8 + if_vers uint8 + reserved1 uint8 + mig_encoding uint8 + int_rep uint8 + char_rep uint8 + float_rep uint8 + reserved2 uint8 +} + +type machport struct { + name uint32 + pad1 uint32 + pad2 uint16 + disposition uint8 + _type uint8 +} + +type stackt struct { + ss_sp *byte + ss_size uintptr + ss_flags int32 +} + +type sigactiont struct { + __sigaction_u [4]byte + sa_tramp unsafe.Pointer + sa_mask uint32 + sa_flags int32 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr *byte + si_value [4]byte + si_band int32 + __pad [7]uint32 +} + +type timeval struct { + tv_sec int32 + tv_usec int32 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type timespec struct { + tv_sec int32 + tv_nsec int32 +} + +type fpcontrol struct { + pad_cgo_0 [2]byte +} + +type fpstatus struct { + pad_cgo_0 [2]byte +} + +type regmmst struct { + mmst_reg [10]int8 + mmst_rsrv [6]int8 +} + +type regxmm struct { + xmm_reg [16]int8 +} + +type regs64 struct { + rax uint64 + rbx uint64 + rcx uint64 + rdx uint64 + rdi uint64 + rsi uint64 + rbp uint64 + rsp uint64 + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + rip uint64 + rflags uint64 + cs uint64 + fs uint64 + gs uint64 +} + +type floatstate64 struct { + fpu_reserved [2]int32 + fpu_fcw fpcontrol + fpu_fsw fpstatus + fpu_ftw uint8 + fpu_rsrv1 uint8 + fpu_fop uint16 + fpu_ip uint32 + fpu_cs uint16 + fpu_rsrv2 uint16 + fpu_dp uint32 + fpu_ds uint16 + fpu_rsrv3 uint16 + fpu_mxcsr uint32 + fpu_mxcsrmask uint32 + fpu_stmm0 regmmst + fpu_stmm1 regmmst + fpu_stmm2 regmmst + fpu_stmm3 regmmst + fpu_stmm4 regmmst + fpu_stmm5 regmmst + fpu_stmm6 regmmst + fpu_stmm7 regmmst + fpu_xmm0 regxmm + fpu_xmm1 regxmm + fpu_xmm2 regxmm + fpu_xmm3 regxmm + fpu_xmm4 regxmm + fpu_xmm5 regxmm + fpu_xmm6 regxmm + fpu_xmm7 regxmm + fpu_xmm8 regxmm + fpu_xmm9 regxmm + fpu_xmm10 regxmm + fpu_xmm11 regxmm + fpu_xmm12 regxmm + fpu_xmm13 regxmm + fpu_xmm14 regxmm + fpu_xmm15 regxmm + fpu_rsrv4 [96]int8 + fpu_reserved1 int32 +} + +type exceptionstate64 struct { + trapno uint16 + cpu uint16 + err uint32 + faultvaddr uint64 +} + +type mcontext64 struct { + es exceptionstate64 + ss regs64 + fs floatstate64 +} + +type regs32 struct { + eax uint32 + ebx uint32 + ecx uint32 + edx uint32 + edi uint32 + esi uint32 + ebp uint32 + esp uint32 + ss uint32 + eflags uint32 + eip uint32 + cs uint32 + ds uint32 + es uint32 + fs uint32 + gs uint32 +} + +type floatstate32 struct { + fpu_reserved [2]int32 + fpu_fcw fpcontrol + fpu_fsw fpstatus + fpu_ftw uint8 + fpu_rsrv1 uint8 + fpu_fop uint16 + fpu_ip uint32 + fpu_cs uint16 + fpu_rsrv2 uint16 + fpu_dp uint32 + fpu_ds uint16 + fpu_rsrv3 uint16 + fpu_mxcsr uint32 + fpu_mxcsrmask uint32 + fpu_stmm0 regmmst + fpu_stmm1 regmmst + fpu_stmm2 regmmst + fpu_stmm3 regmmst + fpu_stmm4 regmmst + fpu_stmm5 regmmst + fpu_stmm6 regmmst + fpu_stmm7 regmmst + fpu_xmm0 regxmm + fpu_xmm1 regxmm + fpu_xmm2 regxmm + fpu_xmm3 regxmm + fpu_xmm4 regxmm + fpu_xmm5 regxmm + fpu_xmm6 regxmm + fpu_xmm7 regxmm + fpu_rsrv4 [224]int8 + fpu_reserved1 int32 +} + +type exceptionstate32 struct { + trapno uint16 + cpu uint16 + err uint32 + faultvaddr uint32 +} + +type mcontext32 struct { + es exceptionstate32 + ss regs32 + fs floatstate32 +} + +type ucontext struct { + uc_onstack int32 + uc_sigmask uint32 + uc_stack stackt + uc_link *ucontext + uc_mcsize uint32 + uc_mcontext *mcontext32 +} + +type keventt struct { + ident uint32 + filter int16 + flags uint16 + fflags uint32 + data int32 + udata *byte +} diff --git a/src/runtime/defs_darwin_386.h b/src/runtime/defs_darwin_386.h deleted file mode 100644 index 0e0b4fbf7..000000000 --- a/src/runtime/defs_darwin_386.h +++ /dev/null @@ -1,392 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_darwin.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_DONTNEED = 0x4, - MADV_FREE = 0x5, - - MACH_MSG_TYPE_MOVE_RECEIVE = 0x10, - MACH_MSG_TYPE_MOVE_SEND = 0x11, - MACH_MSG_TYPE_MOVE_SEND_ONCE = 0x12, - MACH_MSG_TYPE_COPY_SEND = 0x13, - MACH_MSG_TYPE_MAKE_SEND = 0x14, - MACH_MSG_TYPE_MAKE_SEND_ONCE = 0x15, - MACH_MSG_TYPE_COPY_RECEIVE = 0x16, - - MACH_MSG_PORT_DESCRIPTOR = 0x0, - MACH_MSG_OOL_DESCRIPTOR = 0x1, - MACH_MSG_OOL_PORTS_DESCRIPTOR = 0x2, - MACH_MSG_OOL_VOLATILE_DESCRIPTOR = 0x3, - - MACH_MSGH_BITS_COMPLEX = 0x80000000, - - MACH_SEND_MSG = 0x1, - MACH_RCV_MSG = 0x2, - MACH_RCV_LARGE = 0x4, - - MACH_SEND_TIMEOUT = 0x10, - MACH_SEND_INTERRUPT = 0x40, - MACH_SEND_ALWAYS = 0x10000, - MACH_SEND_TRAILER = 0x20000, - MACH_RCV_TIMEOUT = 0x100, - MACH_RCV_NOTIFY = 0x200, - MACH_RCV_INTERRUPT = 0x400, - MACH_RCV_OVERWRITE = 0x1000, - - NDR_PROTOCOL_2_0 = 0x0, - NDR_INT_BIG_ENDIAN = 0x0, - NDR_INT_LITTLE_ENDIAN = 0x1, - NDR_FLOAT_IEEE = 0x0, - NDR_CHAR_ASCII = 0x0, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - SA_USERTRAMP = 0x100, - SA_64REGSET = 0x200, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x7, - FPE_INTOVF = 0x8, - FPE_FLTDIV = 0x1, - FPE_FLTOVF = 0x2, - FPE_FLTUND = 0x3, - FPE_FLTRES = 0x4, - FPE_FLTINV = 0x5, - FPE_FLTSUB = 0x6, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0x40, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct MachBody MachBody; -typedef struct MachHeader MachHeader; -typedef struct MachNDR MachNDR; -typedef struct MachPort MachPort; -typedef struct StackT StackT; -typedef struct SigactionT SigactionT; -typedef struct Siginfo Siginfo; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct Timespec Timespec; -typedef struct FPControl FPControl; -typedef struct FPStatus FPStatus; -typedef struct RegMMST RegMMST; -typedef struct RegXMM RegXMM; -typedef struct Regs64 Regs64; -typedef struct FloatState64 FloatState64; -typedef struct ExceptionState64 ExceptionState64; -typedef struct Mcontext64 Mcontext64; -typedef struct Regs32 Regs32; -typedef struct FloatState32 FloatState32; -typedef struct ExceptionState32 ExceptionState32; -typedef struct Mcontext32 Mcontext32; -typedef struct Ucontext Ucontext; -typedef struct KeventT KeventT; - -#pragma pack on - -struct MachBody { - uint32 msgh_descriptor_count; -}; -struct MachHeader { - uint32 msgh_bits; - uint32 msgh_size; - uint32 msgh_remote_port; - uint32 msgh_local_port; - uint32 msgh_reserved; - int32 msgh_id; -}; -struct MachNDR { - uint8 mig_vers; - uint8 if_vers; - uint8 reserved1; - uint8 mig_encoding; - uint8 int_rep; - uint8 char_rep; - uint8 float_rep; - uint8 reserved2; -}; -struct MachPort { - uint32 name; - uint32 pad1; - uint16 pad2; - uint8 disposition; - uint8 type; -}; - -struct StackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -typedef byte Sighandler[4]; - -struct SigactionT { - byte __sigaction_u[4]; - void *sa_tramp; - uint32 sa_mask; - int32 sa_flags; -}; - -typedef byte Sigval[4]; -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[4]; - int32 si_band; - uint32 __pad[7]; -}; -struct Timeval { - int32 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; -struct Timespec { - int32 tv_sec; - int32 tv_nsec; -}; - -struct FPControl { - byte Pad_cgo_0[2]; -}; -struct FPStatus { - byte Pad_cgo_0[2]; -}; -struct RegMMST { - int8 mmst_reg[10]; - int8 mmst_rsrv[6]; -}; -struct RegXMM { - int8 xmm_reg[16]; -}; - -struct Regs64 { - uint64 rax; - uint64 rbx; - uint64 rcx; - uint64 rdx; - uint64 rdi; - uint64 rsi; - uint64 rbp; - uint64 rsp; - uint64 r8; - uint64 r9; - uint64 r10; - uint64 r11; - uint64 r12; - uint64 r13; - uint64 r14; - uint64 r15; - uint64 rip; - uint64 rflags; - uint64 cs; - uint64 fs; - uint64 gs; -}; -struct FloatState64 { - int32 fpu_reserved[2]; - FPControl fpu_fcw; - FPStatus fpu_fsw; - uint8 fpu_ftw; - uint8 fpu_rsrv1; - uint16 fpu_fop; - uint32 fpu_ip; - uint16 fpu_cs; - uint16 fpu_rsrv2; - uint32 fpu_dp; - uint16 fpu_ds; - uint16 fpu_rsrv3; - uint32 fpu_mxcsr; - uint32 fpu_mxcsrmask; - RegMMST fpu_stmm0; - RegMMST fpu_stmm1; - RegMMST fpu_stmm2; - RegMMST fpu_stmm3; - RegMMST fpu_stmm4; - RegMMST fpu_stmm5; - RegMMST fpu_stmm6; - RegMMST fpu_stmm7; - RegXMM fpu_xmm0; - RegXMM fpu_xmm1; - RegXMM fpu_xmm2; - RegXMM fpu_xmm3; - RegXMM fpu_xmm4; - RegXMM fpu_xmm5; - RegXMM fpu_xmm6; - RegXMM fpu_xmm7; - RegXMM fpu_xmm8; - RegXMM fpu_xmm9; - RegXMM fpu_xmm10; - RegXMM fpu_xmm11; - RegXMM fpu_xmm12; - RegXMM fpu_xmm13; - RegXMM fpu_xmm14; - RegXMM fpu_xmm15; - int8 fpu_rsrv4[96]; - int32 fpu_reserved1; -}; -struct ExceptionState64 { - uint16 trapno; - uint16 cpu; - uint32 err; - uint64 faultvaddr; -}; -struct Mcontext64 { - ExceptionState64 es; - Regs64 ss; - FloatState64 fs; -}; - -struct Regs32 { - uint32 eax; - uint32 ebx; - uint32 ecx; - uint32 edx; - uint32 edi; - uint32 esi; - uint32 ebp; - uint32 esp; - uint32 ss; - uint32 eflags; - uint32 eip; - uint32 cs; - uint32 ds; - uint32 es; - uint32 fs; - uint32 gs; -}; -struct FloatState32 { - int32 fpu_reserved[2]; - FPControl fpu_fcw; - FPStatus fpu_fsw; - uint8 fpu_ftw; - uint8 fpu_rsrv1; - uint16 fpu_fop; - uint32 fpu_ip; - uint16 fpu_cs; - uint16 fpu_rsrv2; - uint32 fpu_dp; - uint16 fpu_ds; - uint16 fpu_rsrv3; - uint32 fpu_mxcsr; - uint32 fpu_mxcsrmask; - RegMMST fpu_stmm0; - RegMMST fpu_stmm1; - RegMMST fpu_stmm2; - RegMMST fpu_stmm3; - RegMMST fpu_stmm4; - RegMMST fpu_stmm5; - RegMMST fpu_stmm6; - RegMMST fpu_stmm7; - RegXMM fpu_xmm0; - RegXMM fpu_xmm1; - RegXMM fpu_xmm2; - RegXMM fpu_xmm3; - RegXMM fpu_xmm4; - RegXMM fpu_xmm5; - RegXMM fpu_xmm6; - RegXMM fpu_xmm7; - int8 fpu_rsrv4[224]; - int32 fpu_reserved1; -}; -struct ExceptionState32 { - uint16 trapno; - uint16 cpu; - uint32 err; - uint32 faultvaddr; -}; -struct Mcontext32 { - ExceptionState32 es; - Regs32 ss; - FloatState32 fs; -}; - -struct Ucontext { - int32 uc_onstack; - uint32 uc_sigmask; - StackT uc_stack; - Ucontext *uc_link; - uint32 uc_mcsize; - Mcontext32 *uc_mcontext; -}; - -struct KeventT { - uint32 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int32 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_darwin_amd64.go b/src/runtime/defs_darwin_amd64.go new file mode 100644 index 000000000..2cd4c0cd0 --- /dev/null +++ b/src/runtime/defs_darwin_amd64.go @@ -0,0 +1,385 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_darwin.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x5 + + _MACH_MSG_TYPE_MOVE_RECEIVE = 0x10 + _MACH_MSG_TYPE_MOVE_SEND = 0x11 + _MACH_MSG_TYPE_MOVE_SEND_ONCE = 0x12 + _MACH_MSG_TYPE_COPY_SEND = 0x13 + _MACH_MSG_TYPE_MAKE_SEND = 0x14 + _MACH_MSG_TYPE_MAKE_SEND_ONCE = 0x15 + _MACH_MSG_TYPE_COPY_RECEIVE = 0x16 + + _MACH_MSG_PORT_DESCRIPTOR = 0x0 + _MACH_MSG_OOL_DESCRIPTOR = 0x1 + _MACH_MSG_OOL_PORTS_DESCRIPTOR = 0x2 + _MACH_MSG_OOL_VOLATILE_DESCRIPTOR = 0x3 + + _MACH_MSGH_BITS_COMPLEX = 0x80000000 + + _MACH_SEND_MSG = 0x1 + _MACH_RCV_MSG = 0x2 + _MACH_RCV_LARGE = 0x4 + + _MACH_SEND_TIMEOUT = 0x10 + _MACH_SEND_INTERRUPT = 0x40 + _MACH_SEND_ALWAYS = 0x10000 + _MACH_SEND_TRAILER = 0x20000 + _MACH_RCV_TIMEOUT = 0x100 + _MACH_RCV_NOTIFY = 0x200 + _MACH_RCV_INTERRUPT = 0x400 + _MACH_RCV_OVERWRITE = 0x1000 + + _NDR_PROTOCOL_2_0 = 0x0 + _NDR_INT_BIG_ENDIAN = 0x0 + _NDR_INT_LITTLE_ENDIAN = 0x1 + _NDR_FLOAT_IEEE = 0x0 + _NDR_CHAR_ASCII = 0x0 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + _SA_USERTRAMP = 0x100 + _SA_64REGSET = 0x200 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x7 + _FPE_INTOVF = 0x8 + _FPE_FLTDIV = 0x1 + _FPE_FLTOVF = 0x2 + _FPE_FLTUND = 0x3 + _FPE_FLTRES = 0x4 + _FPE_FLTINV = 0x5 + _FPE_FLTSUB = 0x6 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0x40 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type machbody struct { + msgh_descriptor_count uint32 +} + +type machheader struct { + msgh_bits uint32 + msgh_size uint32 + msgh_remote_port uint32 + msgh_local_port uint32 + msgh_reserved uint32 + msgh_id int32 +} + +type machndr struct { + mig_vers uint8 + if_vers uint8 + reserved1 uint8 + mig_encoding uint8 + int_rep uint8 + char_rep uint8 + float_rep uint8 + reserved2 uint8 +} + +type machport struct { + name uint32 + pad1 uint32 + pad2 uint16 + disposition uint8 + _type uint8 +} + +type stackt struct { + ss_sp *byte + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigactiont struct { + __sigaction_u [8]byte + sa_tramp unsafe.Pointer + sa_mask uint32 + sa_flags int32 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr *byte + si_value [8]byte + si_band int64 + __pad [7]uint64 +} + +type timeval struct { + tv_sec int64 + tv_usec int32 + pad_cgo_0 [4]byte +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +type fpcontrol struct { + pad_cgo_0 [2]byte +} + +type fpstatus struct { + pad_cgo_0 [2]byte +} + +type regmmst struct { + mmst_reg [10]int8 + mmst_rsrv [6]int8 +} + +type regxmm struct { + xmm_reg [16]int8 +} + +type regs64 struct { + rax uint64 + rbx uint64 + rcx uint64 + rdx uint64 + rdi uint64 + rsi uint64 + rbp uint64 + rsp uint64 + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + rip uint64 + rflags uint64 + cs uint64 + fs uint64 + gs uint64 +} + +type floatstate64 struct { + fpu_reserved [2]int32 + fpu_fcw fpcontrol + fpu_fsw fpstatus + fpu_ftw uint8 + fpu_rsrv1 uint8 + fpu_fop uint16 + fpu_ip uint32 + fpu_cs uint16 + fpu_rsrv2 uint16 + fpu_dp uint32 + fpu_ds uint16 + fpu_rsrv3 uint16 + fpu_mxcsr uint32 + fpu_mxcsrmask uint32 + fpu_stmm0 regmmst + fpu_stmm1 regmmst + fpu_stmm2 regmmst + fpu_stmm3 regmmst + fpu_stmm4 regmmst + fpu_stmm5 regmmst + fpu_stmm6 regmmst + fpu_stmm7 regmmst + fpu_xmm0 regxmm + fpu_xmm1 regxmm + fpu_xmm2 regxmm + fpu_xmm3 regxmm + fpu_xmm4 regxmm + fpu_xmm5 regxmm + fpu_xmm6 regxmm + fpu_xmm7 regxmm + fpu_xmm8 regxmm + fpu_xmm9 regxmm + fpu_xmm10 regxmm + fpu_xmm11 regxmm + fpu_xmm12 regxmm + fpu_xmm13 regxmm + fpu_xmm14 regxmm + fpu_xmm15 regxmm + fpu_rsrv4 [96]int8 + fpu_reserved1 int32 +} + +type exceptionstate64 struct { + trapno uint16 + cpu uint16 + err uint32 + faultvaddr uint64 +} + +type mcontext64 struct { + es exceptionstate64 + ss regs64 + fs floatstate64 + pad_cgo_0 [4]byte +} + +type regs32 struct { + eax uint32 + ebx uint32 + ecx uint32 + edx uint32 + edi uint32 + esi uint32 + ebp uint32 + esp uint32 + ss uint32 + eflags uint32 + eip uint32 + cs uint32 + ds uint32 + es uint32 + fs uint32 + gs uint32 +} + +type floatstate32 struct { + fpu_reserved [2]int32 + fpu_fcw fpcontrol + fpu_fsw fpstatus + fpu_ftw uint8 + fpu_rsrv1 uint8 + fpu_fop uint16 + fpu_ip uint32 + fpu_cs uint16 + fpu_rsrv2 uint16 + fpu_dp uint32 + fpu_ds uint16 + fpu_rsrv3 uint16 + fpu_mxcsr uint32 + fpu_mxcsrmask uint32 + fpu_stmm0 regmmst + fpu_stmm1 regmmst + fpu_stmm2 regmmst + fpu_stmm3 regmmst + fpu_stmm4 regmmst + fpu_stmm5 regmmst + fpu_stmm6 regmmst + fpu_stmm7 regmmst + fpu_xmm0 regxmm + fpu_xmm1 regxmm + fpu_xmm2 regxmm + fpu_xmm3 regxmm + fpu_xmm4 regxmm + fpu_xmm5 regxmm + fpu_xmm6 regxmm + fpu_xmm7 regxmm + fpu_rsrv4 [224]int8 + fpu_reserved1 int32 +} + +type exceptionstate32 struct { + trapno uint16 + cpu uint16 + err uint32 + faultvaddr uint32 +} + +type mcontext32 struct { + es exceptionstate32 + ss regs32 + fs floatstate32 +} + +type ucontext struct { + uc_onstack int32 + uc_sigmask uint32 + uc_stack stackt + uc_link *ucontext + uc_mcsize uint64 + uc_mcontext *mcontext64 +} + +type keventt struct { + ident uint64 + filter int16 + flags uint16 + fflags uint32 + data int64 + udata *byte +} diff --git a/src/runtime/defs_darwin_amd64.h b/src/runtime/defs_darwin_amd64.h deleted file mode 100644 index 4bf83c1cb..000000000 --- a/src/runtime/defs_darwin_amd64.h +++ /dev/null @@ -1,395 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_darwin.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_DONTNEED = 0x4, - MADV_FREE = 0x5, - - MACH_MSG_TYPE_MOVE_RECEIVE = 0x10, - MACH_MSG_TYPE_MOVE_SEND = 0x11, - MACH_MSG_TYPE_MOVE_SEND_ONCE = 0x12, - MACH_MSG_TYPE_COPY_SEND = 0x13, - MACH_MSG_TYPE_MAKE_SEND = 0x14, - MACH_MSG_TYPE_MAKE_SEND_ONCE = 0x15, - MACH_MSG_TYPE_COPY_RECEIVE = 0x16, - - MACH_MSG_PORT_DESCRIPTOR = 0x0, - MACH_MSG_OOL_DESCRIPTOR = 0x1, - MACH_MSG_OOL_PORTS_DESCRIPTOR = 0x2, - MACH_MSG_OOL_VOLATILE_DESCRIPTOR = 0x3, - - MACH_MSGH_BITS_COMPLEX = 0x80000000, - - MACH_SEND_MSG = 0x1, - MACH_RCV_MSG = 0x2, - MACH_RCV_LARGE = 0x4, - - MACH_SEND_TIMEOUT = 0x10, - MACH_SEND_INTERRUPT = 0x40, - MACH_SEND_ALWAYS = 0x10000, - MACH_SEND_TRAILER = 0x20000, - MACH_RCV_TIMEOUT = 0x100, - MACH_RCV_NOTIFY = 0x200, - MACH_RCV_INTERRUPT = 0x400, - MACH_RCV_OVERWRITE = 0x1000, - - NDR_PROTOCOL_2_0 = 0x0, - NDR_INT_BIG_ENDIAN = 0x0, - NDR_INT_LITTLE_ENDIAN = 0x1, - NDR_FLOAT_IEEE = 0x0, - NDR_CHAR_ASCII = 0x0, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - SA_USERTRAMP = 0x100, - SA_64REGSET = 0x200, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x7, - FPE_INTOVF = 0x8, - FPE_FLTDIV = 0x1, - FPE_FLTOVF = 0x2, - FPE_FLTUND = 0x3, - FPE_FLTRES = 0x4, - FPE_FLTINV = 0x5, - FPE_FLTSUB = 0x6, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0x40, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct MachBody MachBody; -typedef struct MachHeader MachHeader; -typedef struct MachNDR MachNDR; -typedef struct MachPort MachPort; -typedef struct StackT StackT; -typedef struct SigactionT SigactionT; -typedef struct Siginfo Siginfo; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct Timespec Timespec; -typedef struct FPControl FPControl; -typedef struct FPStatus FPStatus; -typedef struct RegMMST RegMMST; -typedef struct RegXMM RegXMM; -typedef struct Regs64 Regs64; -typedef struct FloatState64 FloatState64; -typedef struct ExceptionState64 ExceptionState64; -typedef struct Mcontext64 Mcontext64; -typedef struct Regs32 Regs32; -typedef struct FloatState32 FloatState32; -typedef struct ExceptionState32 ExceptionState32; -typedef struct Mcontext32 Mcontext32; -typedef struct Ucontext Ucontext; -typedef struct KeventT KeventT; - -#pragma pack on - -struct MachBody { - uint32 msgh_descriptor_count; -}; -struct MachHeader { - uint32 msgh_bits; - uint32 msgh_size; - uint32 msgh_remote_port; - uint32 msgh_local_port; - uint32 msgh_reserved; - int32 msgh_id; -}; -struct MachNDR { - uint8 mig_vers; - uint8 if_vers; - uint8 reserved1; - uint8 mig_encoding; - uint8 int_rep; - uint8 char_rep; - uint8 float_rep; - uint8 reserved2; -}; -struct MachPort { - uint32 name; - uint32 pad1; - uint16 pad2; - uint8 disposition; - uint8 type; -}; - -struct StackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -typedef byte Sighandler[8]; - -struct SigactionT { - byte __sigaction_u[8]; - void *sa_tramp; - uint32 sa_mask; - int32 sa_flags; -}; - -typedef byte Sigval[8]; -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[8]; - int64 si_band; - uint64 __pad[7]; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; - byte Pad_cgo_0[4]; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; - -struct FPControl { - byte Pad_cgo_0[2]; -}; -struct FPStatus { - byte Pad_cgo_0[2]; -}; -struct RegMMST { - int8 mmst_reg[10]; - int8 mmst_rsrv[6]; -}; -struct RegXMM { - int8 xmm_reg[16]; -}; - -struct Regs64 { - uint64 rax; - uint64 rbx; - uint64 rcx; - uint64 rdx; - uint64 rdi; - uint64 rsi; - uint64 rbp; - uint64 rsp; - uint64 r8; - uint64 r9; - uint64 r10; - uint64 r11; - uint64 r12; - uint64 r13; - uint64 r14; - uint64 r15; - uint64 rip; - uint64 rflags; - uint64 cs; - uint64 fs; - uint64 gs; -}; -struct FloatState64 { - int32 fpu_reserved[2]; - FPControl fpu_fcw; - FPStatus fpu_fsw; - uint8 fpu_ftw; - uint8 fpu_rsrv1; - uint16 fpu_fop; - uint32 fpu_ip; - uint16 fpu_cs; - uint16 fpu_rsrv2; - uint32 fpu_dp; - uint16 fpu_ds; - uint16 fpu_rsrv3; - uint32 fpu_mxcsr; - uint32 fpu_mxcsrmask; - RegMMST fpu_stmm0; - RegMMST fpu_stmm1; - RegMMST fpu_stmm2; - RegMMST fpu_stmm3; - RegMMST fpu_stmm4; - RegMMST fpu_stmm5; - RegMMST fpu_stmm6; - RegMMST fpu_stmm7; - RegXMM fpu_xmm0; - RegXMM fpu_xmm1; - RegXMM fpu_xmm2; - RegXMM fpu_xmm3; - RegXMM fpu_xmm4; - RegXMM fpu_xmm5; - RegXMM fpu_xmm6; - RegXMM fpu_xmm7; - RegXMM fpu_xmm8; - RegXMM fpu_xmm9; - RegXMM fpu_xmm10; - RegXMM fpu_xmm11; - RegXMM fpu_xmm12; - RegXMM fpu_xmm13; - RegXMM fpu_xmm14; - RegXMM fpu_xmm15; - int8 fpu_rsrv4[96]; - int32 fpu_reserved1; -}; -struct ExceptionState64 { - uint16 trapno; - uint16 cpu; - uint32 err; - uint64 faultvaddr; -}; -struct Mcontext64 { - ExceptionState64 es; - Regs64 ss; - FloatState64 fs; - byte Pad_cgo_0[4]; -}; - -struct Regs32 { - uint32 eax; - uint32 ebx; - uint32 ecx; - uint32 edx; - uint32 edi; - uint32 esi; - uint32 ebp; - uint32 esp; - uint32 ss; - uint32 eflags; - uint32 eip; - uint32 cs; - uint32 ds; - uint32 es; - uint32 fs; - uint32 gs; -}; -struct FloatState32 { - int32 fpu_reserved[2]; - FPControl fpu_fcw; - FPStatus fpu_fsw; - uint8 fpu_ftw; - uint8 fpu_rsrv1; - uint16 fpu_fop; - uint32 fpu_ip; - uint16 fpu_cs; - uint16 fpu_rsrv2; - uint32 fpu_dp; - uint16 fpu_ds; - uint16 fpu_rsrv3; - uint32 fpu_mxcsr; - uint32 fpu_mxcsrmask; - RegMMST fpu_stmm0; - RegMMST fpu_stmm1; - RegMMST fpu_stmm2; - RegMMST fpu_stmm3; - RegMMST fpu_stmm4; - RegMMST fpu_stmm5; - RegMMST fpu_stmm6; - RegMMST fpu_stmm7; - RegXMM fpu_xmm0; - RegXMM fpu_xmm1; - RegXMM fpu_xmm2; - RegXMM fpu_xmm3; - RegXMM fpu_xmm4; - RegXMM fpu_xmm5; - RegXMM fpu_xmm6; - RegXMM fpu_xmm7; - int8 fpu_rsrv4[224]; - int32 fpu_reserved1; -}; -struct ExceptionState32 { - uint16 trapno; - uint16 cpu; - uint32 err; - uint32 faultvaddr; -}; -struct Mcontext32 { - ExceptionState32 es; - Regs32 ss; - FloatState32 fs; -}; - -struct Ucontext { - int32 uc_onstack; - uint32 uc_sigmask; - StackT uc_stack; - Ucontext *uc_link; - uint64 uc_mcsize; - Mcontext64 *uc_mcontext; -}; - -struct KeventT { - uint64 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_dragonfly_386.go b/src/runtime/defs_dragonfly_386.go new file mode 100644 index 000000000..1768dbac4 --- /dev/null +++ b/src/runtime/defs_dragonfly_386.go @@ -0,0 +1,190 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_dragonfly.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + _EBUSY = 0x10 + _EAGAIN = 0x23 + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x2 + _FPE_INTOVF = 0x1 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type rtprio struct { + _type uint16 + prio uint16 +} + +type lwpparams struct { + _type unsafe.Pointer + arg *byte + stack *byte + tid1 *int32 + tid2 *int32 +} + +type sigaltstackt struct { + ss_sp *int8 + ss_size uint32 + ss_flags int32 +} + +type sigset struct { + __bits [4]uint32 +} + +type stackt struct { + ss_sp *int8 + ss_size uint32 + ss_flags int32 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr *byte + si_value [4]byte + si_band int32 + __spare__ [7]int32 +} + +type mcontext struct { + mc_onstack int32 + mc_gs int32 + mc_fs int32 + mc_es int32 + mc_ds int32 + mc_edi int32 + mc_esi int32 + mc_ebp int32 + mc_isp int32 + mc_ebx int32 + mc_edx int32 + mc_ecx int32 + mc_eax int32 + mc_xflags int32 + mc_trapno int32 + mc_err int32 + mc_eip int32 + mc_cs int32 + mc_eflags int32 + mc_esp int32 + mc_ss int32 + mc_len int32 + mc_fpformat int32 + mc_ownedfp int32 + mc_fpregs [128]int32 + __spare__ [16]int32 +} + +type ucontext struct { + uc_sigmask sigset + uc_mcontext mcontext + uc_link *ucontext + uc_stack stackt + __spare__ [8]int32 +} + +type timespec struct { + tv_sec int32 + tv_nsec int32 +} + +type timeval struct { + tv_sec int32 + tv_usec int32 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint32 + filter int16 + flags uint16 + fflags uint32 + data int32 + udata *byte +} diff --git a/src/runtime/defs_dragonfly_386.h b/src/runtime/defs_dragonfly_386.h deleted file mode 100644 index f86b9c6b9..000000000 --- a/src/runtime/defs_dragonfly_386.h +++ /dev/null @@ -1,198 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_dragonfly.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - EBUSY = 0x10, - EAGAIN = 0x23, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x2, - FPE_INTOVF = 0x1, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct Rtprio Rtprio; -typedef struct Lwpparams Lwpparams; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct Rtprio { - uint16 type; - uint16 prio; -}; -struct Lwpparams { - void *func; - byte *arg; - byte *stack; - int32 *tid1; - int32 *tid2; -}; -struct SigaltstackT { - int8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct StackT { - int8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[4]; - int32 si_band; - int32 __spare__[7]; -}; - -struct Mcontext { - int32 mc_onstack; - int32 mc_gs; - int32 mc_fs; - int32 mc_es; - int32 mc_ds; - int32 mc_edi; - int32 mc_esi; - int32 mc_ebp; - int32 mc_isp; - int32 mc_ebx; - int32 mc_edx; - int32 mc_ecx; - int32 mc_eax; - int32 mc_xflags; - int32 mc_trapno; - int32 mc_err; - int32 mc_eip; - int32 mc_cs; - int32 mc_eflags; - int32 mc_esp; - int32 mc_ss; - int32 mc_len; - int32 mc_fpformat; - int32 mc_ownedfp; - int32 mc_fpregs[128]; - int32 __spare__[16]; -}; -struct Ucontext { - Sigset uc_sigmask; - Mcontext uc_mcontext; - Ucontext *uc_link; - StackT uc_stack; - int32 __spare__[8]; -}; - -struct Timespec { - int32 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int32 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint32 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int32 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_dragonfly_amd64.go b/src/runtime/defs_dragonfly_amd64.go new file mode 100644 index 000000000..3ac10b090 --- /dev/null +++ b/src/runtime/defs_dragonfly_amd64.go @@ -0,0 +1,208 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_dragonfly.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + _EBUSY = 0x10 + _EAGAIN = 0x23 + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x2 + _FPE_INTOVF = 0x1 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type rtprio struct { + _type uint16 + prio uint16 +} + +type lwpparams struct { + start_func uintptr + arg unsafe.Pointer + stack uintptr + tid1 unsafe.Pointer // *int32 + tid2 unsafe.Pointer // *int32 +} + +type sigaltstackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigset struct { + __bits [4]uint32 +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr uint64 + si_value [8]byte + si_band int64 + __spare__ [7]int32 + pad_cgo_0 [4]byte +} + +type mcontext struct { + mc_onstack uint64 + mc_rdi uint64 + mc_rsi uint64 + mc_rdx uint64 + mc_rcx uint64 + mc_r8 uint64 + mc_r9 uint64 + mc_rax uint64 + mc_rbx uint64 + mc_rbp uint64 + mc_r10 uint64 + mc_r11 uint64 + mc_r12 uint64 + mc_r13 uint64 + mc_r14 uint64 + mc_r15 uint64 + mc_xflags uint64 + mc_trapno uint64 + mc_addr uint64 + mc_flags uint64 + mc_err uint64 + mc_rip uint64 + mc_cs uint64 + mc_rflags uint64 + mc_rsp uint64 + mc_ss uint64 + mc_len uint32 + mc_fpformat uint32 + mc_ownedfp uint32 + mc_reserved uint32 + mc_unused [8]uint32 + mc_fpregs [256]int32 +} + +type ucontext struct { + uc_sigmask sigset + pad_cgo_0 [48]byte + uc_mcontext mcontext + uc_link *ucontext + uc_stack stackt + __spare__ [8]int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint64 + filter int16 + flags uint16 + fflags uint32 + data int64 + udata *byte +} diff --git a/src/runtime/defs_dragonfly_amd64.h b/src/runtime/defs_dragonfly_amd64.h deleted file mode 100644 index 671555241..000000000 --- a/src/runtime/defs_dragonfly_amd64.h +++ /dev/null @@ -1,208 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_dragonfly.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - EBUSY = 0x10, - EAGAIN = 0x23, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x2, - FPE_INTOVF = 0x1, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct Rtprio Rtprio; -typedef struct Lwpparams Lwpparams; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct Rtprio { - uint16 type; - uint16 prio; -}; -struct Lwpparams { - void *func; - byte *arg; - byte *stack; - int32 *tid1; - int32 *tid2; -}; -struct SigaltstackT { - int8 *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct StackT { - int8 *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; - -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[8]; - int64 si_band; - int32 __spare__[7]; - byte Pad_cgo_0[4]; -}; - -struct Mcontext { - int64 mc_onstack; - int64 mc_rdi; - int64 mc_rsi; - int64 mc_rdx; - int64 mc_rcx; - int64 mc_r8; - int64 mc_r9; - int64 mc_rax; - int64 mc_rbx; - int64 mc_rbp; - int64 mc_r10; - int64 mc_r11; - int64 mc_r12; - int64 mc_r13; - int64 mc_r14; - int64 mc_r15; - int64 mc_xflags; - int64 mc_trapno; - int64 mc_addr; - int64 mc_flags; - int64 mc_err; - int64 mc_rip; - int64 mc_cs; - int64 mc_rflags; - int64 mc_rsp; - int64 mc_ss; - uint32 mc_len; - uint32 mc_fpformat; - uint32 mc_ownedfp; - uint32 mc_reserved; - uint32 mc_unused[8]; - int32 mc_fpregs[256]; -}; -struct Ucontext { - Sigset uc_sigmask; - byte Pad_cgo_0[48]; - Mcontext uc_mcontext; - Ucontext *uc_link; - StackT uc_stack; - int32 __spare__[8]; -}; - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int64 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint64 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_freebsd_386.go b/src/runtime/defs_freebsd_386.go new file mode 100644 index 000000000..6938c1873 --- /dev/null +++ b/src/runtime/defs_freebsd_386.go @@ -0,0 +1,213 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_freebsd.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _UMTX_OP_WAIT_UINT = 0xb + _UMTX_OP_WAIT_UINT_PRIVATE = 0xf + _UMTX_OP_WAKE = 0x3 + _UMTX_OP_WAKE_PRIVATE = 0x10 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x2 + _FPE_INTOVF = 0x1 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0x40 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type rtprio struct { + _type uint16 + prio uint16 +} + +type thrparam struct { + start_func uintptr + arg unsafe.Pointer + stack_base uintptr + stack_size uintptr + tls_base unsafe.Pointer + tls_size uintptr + child_tid unsafe.Pointer // *int32 + parent_tid *int32 + flags int32 + rtp *rtprio + spare [3]uintptr +} + +type sigaltstackt struct { + ss_sp *int8 + ss_size uint32 + ss_flags int32 +} + +type sigset struct { + __bits [4]uint32 +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr uintptr + si_value [4]byte + _reason [32]byte +} + +type mcontext struct { + mc_onstack uint32 + mc_gs uint32 + mc_fs uint32 + mc_es uint32 + mc_ds uint32 + mc_edi uint32 + mc_esi uint32 + mc_ebp uint32 + mc_isp uint32 + mc_ebx uint32 + mc_edx uint32 + mc_ecx uint32 + mc_eax uint32 + mc_trapno uint32 + mc_err uint32 + mc_eip uint32 + mc_cs uint32 + mc_eflags uint32 + mc_esp uint32 + mc_ss uint32 + mc_len uint32 + mc_fpformat uint32 + mc_ownedfp uint32 + mc_flags uint32 + mc_fpstate [128]uint32 + mc_fsbase uint32 + mc_gsbase uint32 + mc_xfpustate uint32 + mc_xfpustate_len uint32 + mc_spare2 [4]uint32 +} + +type ucontext struct { + uc_sigmask sigset + uc_mcontext mcontext + uc_link *ucontext + uc_stack stackt + uc_flags int32 + __spare__ [4]int32 + pad_cgo_0 [12]byte +} + +type timespec struct { + tv_sec int32 + tv_nsec int32 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = int32(x) +} + +type timeval struct { + tv_sec int32 + tv_usec int32 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint32 + filter int16 + flags uint16 + fflags uint32 + data int32 + udata *byte +} diff --git a/src/runtime/defs_freebsd_386.h b/src/runtime/defs_freebsd_386.h deleted file mode 100644 index 156dccba4..000000000 --- a/src/runtime/defs_freebsd_386.h +++ /dev/null @@ -1,213 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_freebsd.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - UMTX_OP_WAIT_UINT = 0xb, - UMTX_OP_WAIT_UINT_PRIVATE = 0xf, - UMTX_OP_WAKE = 0x3, - UMTX_OP_WAKE_PRIVATE = 0x10, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x2, - FPE_INTOVF = 0x1, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0x40, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct Rtprio Rtprio; -typedef struct ThrParam ThrParam; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct Rtprio { - uint16 type; - uint16 prio; -}; -struct ThrParam { - void *start_func; - byte *arg; - int8 *stack_base; - uint32 stack_size; - int8 *tls_base; - uint32 tls_size; - int32 *child_tid; - int32 *parent_tid; - int32 flags; - Rtprio *rtp; - void *spare[3]; -}; -struct SigaltstackT { - int8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct StackT { - int8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[4]; - byte _reason[32]; -}; - -struct Mcontext { - int32 mc_onstack; - int32 mc_gs; - int32 mc_fs; - int32 mc_es; - int32 mc_ds; - int32 mc_edi; - int32 mc_esi; - int32 mc_ebp; - int32 mc_isp; - int32 mc_ebx; - int32 mc_edx; - int32 mc_ecx; - int32 mc_eax; - int32 mc_trapno; - int32 mc_err; - int32 mc_eip; - int32 mc_cs; - int32 mc_eflags; - int32 mc_esp; - int32 mc_ss; - int32 mc_len; - int32 mc_fpformat; - int32 mc_ownedfp; - int32 mc_flags; - int32 mc_fpstate[128]; - int32 mc_fsbase; - int32 mc_gsbase; - int32 mc_xfpustate; - int32 mc_xfpustate_len; - int32 mc_spare2[4]; -}; -struct Ucontext { - Sigset uc_sigmask; - Mcontext uc_mcontext; - Ucontext *uc_link; - StackT uc_stack; - int32 uc_flags; - int32 __spare__[4]; - byte Pad_cgo_0[12]; -}; - -struct Timespec { - int32 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int32 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint32 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int32 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_freebsd_amd64.go b/src/runtime/defs_freebsd_amd64.go new file mode 100644 index 000000000..de98e7a3c --- /dev/null +++ b/src/runtime/defs_freebsd_amd64.go @@ -0,0 +1,224 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_freebsd.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _UMTX_OP_WAIT_UINT = 0xb + _UMTX_OP_WAIT_UINT_PRIVATE = 0xf + _UMTX_OP_WAKE = 0x3 + _UMTX_OP_WAKE_PRIVATE = 0x10 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x2 + _FPE_INTOVF = 0x1 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0x40 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type rtprio struct { + _type uint16 + prio uint16 +} + +type thrparam struct { + start_func uintptr + arg unsafe.Pointer + stack_base uintptr + stack_size uintptr + tls_base unsafe.Pointer + tls_size uintptr + child_tid unsafe.Pointer // *int64 + parent_tid *int64 + flags int32 + pad_cgo_0 [4]byte + rtp *rtprio + spare [3]uintptr +} + +type sigaltstackt struct { + ss_sp *int8 + ss_size uint64 + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigset struct { + __bits [4]uint32 +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr uint64 + si_value [8]byte + _reason [40]byte +} + +type mcontext struct { + mc_onstack uint64 + mc_rdi uint64 + mc_rsi uint64 + mc_rdx uint64 + mc_rcx uint64 + mc_r8 uint64 + mc_r9 uint64 + mc_rax uint64 + mc_rbx uint64 + mc_rbp uint64 + mc_r10 uint64 + mc_r11 uint64 + mc_r12 uint64 + mc_r13 uint64 + mc_r14 uint64 + mc_r15 uint64 + mc_trapno uint32 + mc_fs uint16 + mc_gs uint16 + mc_addr uint64 + mc_flags uint32 + mc_es uint16 + mc_ds uint16 + mc_err uint64 + mc_rip uint64 + mc_cs uint64 + mc_rflags uint64 + mc_rsp uint64 + mc_ss uint64 + mc_len uint64 + mc_fpformat uint64 + mc_ownedfp uint64 + mc_fpstate [64]uint64 + mc_fsbase uint64 + mc_gsbase uint64 + mc_xfpustate uint64 + mc_xfpustate_len uint64 + mc_spare [4]uint64 +} + +type ucontext struct { + uc_sigmask sigset + uc_mcontext mcontext + uc_link *ucontext + uc_stack stackt + uc_flags int32 + __spare__ [4]int32 + pad_cgo_0 [12]byte +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint64 + filter int16 + flags uint16 + fflags uint32 + data int64 + udata *byte +} diff --git a/src/runtime/defs_freebsd_amd64.h b/src/runtime/defs_freebsd_amd64.h deleted file mode 100644 index 4ba8956a2..000000000 --- a/src/runtime/defs_freebsd_amd64.h +++ /dev/null @@ -1,224 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_freebsd.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - UMTX_OP_WAIT_UINT = 0xb, - UMTX_OP_WAIT_UINT_PRIVATE = 0xf, - UMTX_OP_WAKE = 0x3, - UMTX_OP_WAKE_PRIVATE = 0x10, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x2, - FPE_INTOVF = 0x1, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0x40, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct Rtprio Rtprio; -typedef struct ThrParam ThrParam; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct Rtprio { - uint16 type; - uint16 prio; -}; -struct ThrParam { - void *start_func; - byte *arg; - int8 *stack_base; - uint64 stack_size; - int8 *tls_base; - uint64 tls_size; - int64 *child_tid; - int64 *parent_tid; - int32 flags; - byte Pad_cgo_0[4]; - Rtprio *rtp; - void *spare[3]; -}; -struct SigaltstackT { - int8 *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct StackT { - int8 *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; - -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[8]; - byte _reason[40]; -}; - -struct Mcontext { - int64 mc_onstack; - int64 mc_rdi; - int64 mc_rsi; - int64 mc_rdx; - int64 mc_rcx; - int64 mc_r8; - int64 mc_r9; - int64 mc_rax; - int64 mc_rbx; - int64 mc_rbp; - int64 mc_r10; - int64 mc_r11; - int64 mc_r12; - int64 mc_r13; - int64 mc_r14; - int64 mc_r15; - uint32 mc_trapno; - uint16 mc_fs; - uint16 mc_gs; - int64 mc_addr; - uint32 mc_flags; - uint16 mc_es; - uint16 mc_ds; - int64 mc_err; - int64 mc_rip; - int64 mc_cs; - int64 mc_rflags; - int64 mc_rsp; - int64 mc_ss; - int64 mc_len; - int64 mc_fpformat; - int64 mc_ownedfp; - int64 mc_fpstate[64]; - int64 mc_fsbase; - int64 mc_gsbase; - int64 mc_xfpustate; - int64 mc_xfpustate_len; - int64 mc_spare[4]; -}; -struct Ucontext { - Sigset uc_sigmask; - Mcontext uc_mcontext; - Ucontext *uc_link; - StackT uc_stack; - int32 uc_flags; - int32 __spare__[4]; - byte Pad_cgo_0[12]; -}; - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int64 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint64 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_freebsd_arm.go b/src/runtime/defs_freebsd_arm.go new file mode 100644 index 000000000..744330f4b --- /dev/null +++ b/src/runtime/defs_freebsd_arm.go @@ -0,0 +1,186 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_freebsd.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x5 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _UMTX_OP_WAIT_UINT = 0xb + _UMTX_OP_WAIT_UINT_PRIVATE = 0xf + _UMTX_OP_WAKE = 0x3 + _UMTX_OP_WAKE_PRIVATE = 0x10 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x2 + _FPE_INTOVF = 0x1 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_RECEIPT = 0x40 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type rtprio struct { + _type uint16 + prio uint16 +} + +type thrparam struct { + start_func uintptr + arg unsafe.Pointer + stack_base uintptr + stack_size uintptr + tls_base unsafe.Pointer + tls_size uintptr + child_tid unsafe.Pointer // *int32 + parent_tid *int32 + flags int32 + rtp *rtprio + spare [3]uintptr +} + +type sigaltstackt struct { + ss_sp *uint8 + ss_size uint32 + ss_flags int32 +} + +type sigset struct { + __bits [4]uint32 +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr uintptr + si_value [4]byte + _reason [32]byte +} + +type mcontext struct { + __gregs [17]uint32 + __fpu [140]byte +} + +type ucontext struct { + uc_sigmask sigset + uc_mcontext mcontext + uc_link *ucontext + uc_stack stackt + uc_flags int32 + __spare__ [4]int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int32 + pad_cgo_0 [4]byte +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +type timeval struct { + tv_sec int64 + tv_usec int32 + pad_cgo_0 [4]byte +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint32 + filter int16 + flags uint16 + fflags uint32 + data int32 + udata *byte +} diff --git a/src/runtime/defs_freebsd_arm.h b/src/runtime/defs_freebsd_arm.h deleted file mode 100644 index 17deba68d..000000000 --- a/src/runtime/defs_freebsd_arm.h +++ /dev/null @@ -1,186 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_freebsd.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - UMTX_OP_WAIT_UINT = 0xb, - UMTX_OP_WAIT_UINT_PRIVATE = 0xf, - UMTX_OP_WAKE = 0x3, - UMTX_OP_WAKE_PRIVATE = 0x10, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x2, - FPE_INTOVF = 0x1, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0x40, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct Rtprio Rtprio; -typedef struct ThrParam ThrParam; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct Rtprio { - uint16 type; - uint16 prio; -}; -struct ThrParam { - void *start_func; - byte *arg; - uint8 *stack_base; - uint32 stack_size; - uint8 *tls_base; - uint32 tls_size; - int32 *child_tid; - int32 *parent_tid; - int32 flags; - Rtprio *rtp; - void *spare[3]; -}; -struct SigaltstackT { - uint8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct StackT { - uint8 *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - int32 si_pid; - uint32 si_uid; - int32 si_status; - byte *si_addr; - byte si_value[4]; - byte _reason[32]; -}; - -struct Mcontext { - uint32 __gregs[17]; - byte __fpu[140]; -}; -struct Ucontext { - Sigset uc_sigmask; - Mcontext uc_mcontext; - Ucontext *uc_link; - StackT uc_stack; - int32 uc_flags; - int32 __spare__[4]; -}; - -struct Timespec { - int64 tv_sec; - int32 tv_nsec; - byte Pad_cgo_0[4]; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; - byte Pad_cgo_0[4]; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint32 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int32 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_linux.go b/src/runtime/defs_linux.go index 8657dbb0e..553366a50 100644 --- a/src/runtime/defs_linux.go +++ b/src/runtime/defs_linux.go @@ -20,6 +20,7 @@ package runtime // headers for things like ucontext_t, so that happens in // a separate file, defs1.go. +#define _SYS_TYPES_H // avoid inclusion of sys/types.h #include <asm/posix_types.h> #define size_t __kernel_size_t #include <asm/signal.h> @@ -28,7 +29,7 @@ package runtime #include <asm-generic/errno.h> #include <asm-generic/poll.h> #include <linux/eventpoll.h> -#undef size_t +#include <linux/time.h> */ import "C" @@ -48,10 +49,9 @@ const ( MADV_DONTNEED = C.MADV_DONTNEED - SA_RESTART = C.SA_RESTART - SA_ONSTACK = C.SA_ONSTACK - SA_RESTORER = C.SA_RESTORER - SA_SIGINFO = C.SA_SIGINFO + SA_RESTART = C.SA_RESTART + SA_ONSTACK = C.SA_ONSTACK + SA_SIGINFO = C.SA_SIGINFO SIGHUP = C.SIGHUP SIGINT = C.SIGINT @@ -116,6 +116,7 @@ const ( EPOLL_CTL_MOD = C.EPOLL_CTL_MOD ) +type Sigset C.sigset_t type Timespec C.struct_timespec type Timeval C.struct_timeval type Sigaction C.struct_sigaction diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go new file mode 100644 index 000000000..f55924b61 --- /dev/null +++ b/src/runtime/defs_linux_386.go @@ -0,0 +1,218 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs2_linux.go + +package runtime + +const ( + _EINTR = 0x4 + _EAGAIN = 0xb + _ENOMEM = 0xc + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x20 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + + _SA_RESTART = 0x10000000 + _SA_ONSTACK = 0x8000000 + _SA_RESTORER = 0x4000000 + _SA_SIGINFO = 0x4 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0xa + _SIGSEGV = 0xb + _SIGUSR2 = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGSTKFLT = 0x10 + _SIGCHLD = 0x11 + _SIGCONT = 0x12 + _SIGSTOP = 0x13 + _SIGTSTP = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGIO = 0x1d + _SIGPWR = 0x1e + _SIGSYS = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _O_RDONLY = 0x0 + _O_CLOEXEC = 0x80000 + + _EPOLLIN = 0x1 + _EPOLLOUT = 0x4 + _EPOLLERR = 0x8 + _EPOLLHUP = 0x10 + _EPOLLRDHUP = 0x2000 + _EPOLLET = 0x80000000 + _EPOLL_CLOEXEC = 0x80000 + _EPOLL_CTL_ADD = 0x1 + _EPOLL_CTL_DEL = 0x2 + _EPOLL_CTL_MOD = 0x3 +) + +type fpreg struct { + significand [4]uint16 + exponent uint16 +} + +type fpxreg struct { + significand [4]uint16 + exponent uint16 + padding [3]uint16 +} + +type xmmreg struct { + element [4]uint32 +} + +type fpstate struct { + cw uint32 + sw uint32 + tag uint32 + ipoff uint32 + cssel uint32 + dataoff uint32 + datasel uint32 + _st [8]fpreg + status uint16 + magic uint16 + _fxsr_env [6]uint32 + mxcsr uint32 + reserved uint32 + _fxsr_st [8]fpxreg + _xmm [8]xmmreg + padding1 [44]uint32 + anon0 [48]byte +} + +type timespec struct { + tv_sec int32 + tv_nsec int32 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = int32(x) +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = x +} + +type timeval struct { + tv_sec int32 + tv_usec int32 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type sigactiont struct { + sa_handler uintptr + sa_flags uint32 + sa_restorer uintptr + sa_mask uint64 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + // below here is a union; si_addr is the only field we use + si_addr uint32 +} + +type sigaltstackt struct { + ss_sp *byte + ss_flags int32 + ss_size uintptr +} + +type sigcontext struct { + gs uint16 + __gsh uint16 + fs uint16 + __fsh uint16 + es uint16 + __esh uint16 + ds uint16 + __dsh uint16 + edi uint32 + esi uint32 + ebp uint32 + esp uint32 + ebx uint32 + edx uint32 + ecx uint32 + eax uint32 + trapno uint32 + err uint32 + eip uint32 + cs uint16 + __csh uint16 + eflags uint32 + esp_at_signal uint32 + ss uint16 + __ssh uint16 + fpstate *fpstate + oldmask uint32 + cr2 uint32 +} + +type ucontext struct { + uc_flags uint32 + uc_link *ucontext + uc_stack sigaltstackt + uc_mcontext sigcontext + uc_sigmask uint32 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type epollevent struct { + events uint32 + data [8]byte // to match amd64 +} diff --git a/src/runtime/defs_linux_386.h b/src/runtime/defs_linux_386.h deleted file mode 100644 index 24a05d862..000000000 --- a/src/runtime/defs_linux_386.h +++ /dev/null @@ -1,211 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs2_linux.go - - -enum { - EINTR = 0x4, - EAGAIN = 0xb, - ENOMEM = 0xc, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x20, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_DONTNEED = 0x4, - - SA_RESTART = 0x10000000, - SA_ONSTACK = 0x8000000, - SA_RESTORER = 0x4000000, - SA_SIGINFO = 0x4, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGBUS = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGUSR1 = 0xa, - SIGSEGV = 0xb, - SIGUSR2 = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGSTKFLT = 0x10, - SIGCHLD = 0x11, - SIGCONT = 0x12, - SIGSTOP = 0x13, - SIGTSTP = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGURG = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGIO = 0x1d, - SIGPWR = 0x1e, - SIGSYS = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - O_RDONLY = 0x0, - O_CLOEXEC = 0x80000, - - EPOLLIN = 0x1, - EPOLLOUT = 0x4, - EPOLLERR = 0x8, - EPOLLHUP = 0x10, - EPOLLRDHUP = 0x2000, - EPOLLET = -0x80000000, - EPOLL_CLOEXEC = 0x80000, - EPOLL_CTL_ADD = 0x1, - EPOLL_CTL_DEL = 0x2, - EPOLL_CTL_MOD = 0x3, -}; - -typedef struct Fpreg Fpreg; -typedef struct Fpxreg Fpxreg; -typedef struct Xmmreg Xmmreg; -typedef struct Fpstate Fpstate; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct SigactionT SigactionT; -typedef struct Siginfo Siginfo; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigcontext Sigcontext; -typedef struct Ucontext Ucontext; -typedef struct Itimerval Itimerval; -typedef struct EpollEvent EpollEvent; - -#pragma pack on - -struct Fpreg { - uint16 significand[4]; - uint16 exponent; -}; -struct Fpxreg { - uint16 significand[4]; - uint16 exponent; - uint16 padding[3]; -}; -struct Xmmreg { - uint32 element[4]; -}; -struct Fpstate { - uint32 cw; - uint32 sw; - uint32 tag; - uint32 ipoff; - uint32 cssel; - uint32 dataoff; - uint32 datasel; - Fpreg _st[8]; - uint16 status; - uint16 magic; - uint32 _fxsr_env[6]; - uint32 mxcsr; - uint32 reserved; - Fpxreg _fxsr_st[8]; - Xmmreg _xmm[8]; - uint32 padding1[44]; - byte anon0[48]; -}; -struct Timespec { - int32 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int32 tv_sec; - int32 tv_usec; -}; -struct SigactionT { - void *k_sa_handler; - uint32 sa_flags; - void *sa_restorer; - uint64 sa_mask; -}; -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - byte _sifields[116]; -}; -struct SigaltstackT { - byte *ss_sp; - int32 ss_flags; - uint32 ss_size; -}; -struct Sigcontext { - uint16 gs; - uint16 __gsh; - uint16 fs; - uint16 __fsh; - uint16 es; - uint16 __esh; - uint16 ds; - uint16 __dsh; - uint32 edi; - uint32 esi; - uint32 ebp; - uint32 esp; - uint32 ebx; - uint32 edx; - uint32 ecx; - uint32 eax; - uint32 trapno; - uint32 err; - uint32 eip; - uint16 cs; - uint16 __csh; - uint32 eflags; - uint32 esp_at_signal; - uint16 ss; - uint16 __ssh; - Fpstate *fpstate; - uint32 oldmask; - uint32 cr2; -}; -struct Ucontext { - uint32 uc_flags; - Ucontext *uc_link; - SigaltstackT uc_stack; - Sigcontext uc_mcontext; - uint32 uc_sigmask; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; -struct EpollEvent { - uint32 events; - byte data[8]; // to match amd64 -}; - - -#pragma pack off diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go new file mode 100644 index 000000000..a73f47514 --- /dev/null +++ b/src/runtime/defs_linux_amd64.go @@ -0,0 +1,253 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs1_linux.go + +package runtime + +const ( + _EINTR = 0x4 + _EAGAIN = 0xb + _ENOMEM = 0xc + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x20 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + + _SA_RESTART = 0x10000000 + _SA_ONSTACK = 0x8000000 + _SA_RESTORER = 0x4000000 + _SA_SIGINFO = 0x4 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0xa + _SIGSEGV = 0xb + _SIGUSR2 = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGSTKFLT = 0x10 + _SIGCHLD = 0x11 + _SIGCONT = 0x12 + _SIGSTOP = 0x13 + _SIGTSTP = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGIO = 0x1d + _SIGPWR = 0x1e + _SIGSYS = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EPOLLIN = 0x1 + _EPOLLOUT = 0x4 + _EPOLLERR = 0x8 + _EPOLLHUP = 0x10 + _EPOLLRDHUP = 0x2000 + _EPOLLET = 0x80000000 + _EPOLL_CLOEXEC = 0x80000 + _EPOLL_CTL_ADD = 0x1 + _EPOLL_CTL_DEL = 0x2 + _EPOLL_CTL_MOD = 0x3 +) + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = int64(x) +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type sigactiont struct { + sa_handler uintptr + sa_flags uint64 + sa_restorer uintptr + sa_mask uint64 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + // below here is a union; si_addr is the only field we use + si_addr uint64 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type epollevent struct { + events uint32 + data [8]byte // unaligned uintptr +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs1_linux.go + +const ( + _O_RDONLY = 0x0 + _O_CLOEXEC = 0x80000 +) + +type usigset struct { + __val [16]uint64 +} + +type fpxreg struct { + significand [4]uint16 + exponent uint16 + padding [3]uint16 +} + +type xmmreg struct { + element [4]uint32 +} + +type fpstate struct { + cwd uint16 + swd uint16 + ftw uint16 + fop uint16 + rip uint64 + rdp uint64 + mxcsr uint32 + mxcr_mask uint32 + _st [8]fpxreg + _xmm [16]xmmreg + padding [24]uint32 +} + +type fpxreg1 struct { + significand [4]uint16 + exponent uint16 + padding [3]uint16 +} + +type xmmreg1 struct { + element [4]uint32 +} + +type fpstate1 struct { + cwd uint16 + swd uint16 + ftw uint16 + fop uint16 + rip uint64 + rdp uint64 + mxcsr uint32 + mxcr_mask uint32 + _st [8]fpxreg1 + _xmm [16]xmmreg1 + padding [24]uint32 +} + +type fpreg1 struct { + significand [4]uint16 + exponent uint16 +} + +type sigaltstackt struct { + ss_sp *byte + ss_flags int32 + pad_cgo_0 [4]byte + ss_size uintptr +} + +type mcontext struct { + gregs [23]uint64 + fpregs *fpstate + __reserved1 [8]uint64 +} + +type ucontext struct { + uc_flags uint64 + uc_link *ucontext + uc_stack sigaltstackt + uc_mcontext mcontext + uc_sigmask usigset + __fpregs_mem fpstate +} + +type sigcontext struct { + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + rdi uint64 + rsi uint64 + rbp uint64 + rbx uint64 + rdx uint64 + rax uint64 + rcx uint64 + rsp uint64 + rip uint64 + eflags uint64 + cs uint16 + gs uint16 + fs uint16 + __pad0 uint16 + err uint64 + trapno uint64 + oldmask uint64 + cr2 uint64 + fpstate *fpstate1 + __reserved1 [8]uint64 +} diff --git a/src/runtime/defs_linux_amd64.h b/src/runtime/defs_linux_amd64.h deleted file mode 100644 index 14616dffe..000000000 --- a/src/runtime/defs_linux_amd64.h +++ /dev/null @@ -1,254 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_linux.go defs1_linux.go - - -enum { - EINTR = 0x4, - EAGAIN = 0xb, - ENOMEM = 0xc, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x20, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_DONTNEED = 0x4, - - SA_RESTART = 0x10000000, - SA_ONSTACK = 0x8000000, - SA_RESTORER = 0x4000000, - SA_SIGINFO = 0x4, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGBUS = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGUSR1 = 0xa, - SIGSEGV = 0xb, - SIGUSR2 = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGSTKFLT = 0x10, - SIGCHLD = 0x11, - SIGCONT = 0x12, - SIGSTOP = 0x13, - SIGTSTP = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGURG = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGIO = 0x1d, - SIGPWR = 0x1e, - SIGSYS = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EPOLLIN = 0x1, - EPOLLOUT = 0x4, - EPOLLERR = 0x8, - EPOLLHUP = 0x10, - EPOLLRDHUP = 0x2000, - EPOLLET = -0x80000000, - EPOLL_CLOEXEC = 0x80000, - EPOLL_CTL_ADD = 0x1, - EPOLL_CTL_DEL = 0x2, - EPOLL_CTL_MOD = 0x3, -}; - -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct SigactionT SigactionT; -typedef struct Siginfo Siginfo; -typedef struct Itimerval Itimerval; -typedef struct EpollEvent EpollEvent; - -#pragma pack on - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int64 tv_usec; -}; -struct SigactionT { - void *sa_handler; - uint64 sa_flags; - void *sa_restorer; - uint64 sa_mask; -}; -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - byte Pad_cgo_0[4]; - byte _sifields[112]; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; -struct EpollEvent { - uint32 events; - byte data[8]; // unaligned uintptr -}; - - -#pragma pack off -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_linux.go defs1_linux.go - - -enum { - O_RDONLY = 0x0, - O_CLOEXEC = 0x80000, -}; - -typedef struct Usigset Usigset; -typedef struct Fpxreg Fpxreg; -typedef struct Xmmreg Xmmreg; -typedef struct Fpstate Fpstate; -typedef struct Fpxreg1 Fpxreg1; -typedef struct Xmmreg1 Xmmreg1; -typedef struct Fpstate1 Fpstate1; -typedef struct Fpreg1 Fpreg1; -typedef struct SigaltstackT SigaltstackT; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Sigcontext Sigcontext; - -#pragma pack on - -struct Usigset { - uint64 __val[16]; -}; -struct Fpxreg { - uint16 significand[4]; - uint16 exponent; - uint16 padding[3]; -}; -struct Xmmreg { - uint32 element[4]; -}; -struct Fpstate { - uint16 cwd; - uint16 swd; - uint16 ftw; - uint16 fop; - uint64 rip; - uint64 rdp; - uint32 mxcsr; - uint32 mxcr_mask; - Fpxreg _st[8]; - Xmmreg _xmm[16]; - uint32 padding[24]; -}; -struct Fpxreg1 { - uint16 significand[4]; - uint16 exponent; - uint16 padding[3]; -}; -struct Xmmreg1 { - uint32 element[4]; -}; -struct Fpstate1 { - uint16 cwd; - uint16 swd; - uint16 ftw; - uint16 fop; - uint64 rip; - uint64 rdp; - uint32 mxcsr; - uint32 mxcr_mask; - Fpxreg1 _st[8]; - Xmmreg1 _xmm[16]; - uint32 padding[24]; -}; -struct Fpreg1 { - uint16 significand[4]; - uint16 exponent; -}; -struct SigaltstackT { - byte *ss_sp; - int32 ss_flags; - byte Pad_cgo_0[4]; - uint64 ss_size; -}; -struct Mcontext { - int64 gregs[23]; - Fpstate *fpregs; - uint64 __reserved1[8]; -}; -struct Ucontext { - uint64 uc_flags; - Ucontext *uc_link; - SigaltstackT uc_stack; - Mcontext uc_mcontext; - Usigset uc_sigmask; - Fpstate __fpregs_mem; -}; -struct Sigcontext { - uint64 r8; - uint64 r9; - uint64 r10; - uint64 r11; - uint64 r12; - uint64 r13; - uint64 r14; - uint64 r15; - uint64 rdi; - uint64 rsi; - uint64 rbp; - uint64 rbx; - uint64 rdx; - uint64 rax; - uint64 rcx; - uint64 rsp; - uint64 rip; - uint64 eflags; - uint16 cs; - uint16 gs; - uint16 fs; - uint16 __pad0; - uint64 err; - uint64 trapno; - uint64 oldmask; - uint64 cr2; - Fpstate1 *fpstate; - uint64 __reserved1[8]; -}; - - -#pragma pack off diff --git a/src/runtime/defs_linux_arm.go b/src/runtime/defs_linux_arm.go new file mode 100644 index 000000000..c3a6e2f01 --- /dev/null +++ b/src/runtime/defs_linux_arm.go @@ -0,0 +1,168 @@ +package runtime + +// Constants +const ( + _EINTR = 0x4 + _ENOMEM = 0xc + _EAGAIN = 0xb + + _PROT_NONE = 0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + _MAP_ANON = 0x20 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + _MADV_DONTNEED = 0x4 + _SA_RESTART = 0x10000000 + _SA_ONSTACK = 0x8000000 + _SA_RESTORER = 0 // unused on ARM + _SA_SIGINFO = 0x4 + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0xa + _SIGSEGV = 0xb + _SIGUSR2 = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGSTKFLT = 0x10 + _SIGCHLD = 0x11 + _SIGCONT = 0x12 + _SIGSTOP = 0x13 + _SIGTSTP = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGIO = 0x1d + _SIGPWR = 0x1e + _SIGSYS = 0x1f + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + _ITIMER_REAL = 0 + _ITIMER_PROF = 0x2 + _ITIMER_VIRTUAL = 0x1 + _O_RDONLY = 0 + _O_CLOEXEC = 02000000 + + _EPOLLIN = 0x1 + _EPOLLOUT = 0x4 + _EPOLLERR = 0x8 + _EPOLLHUP = 0x10 + _EPOLLRDHUP = 0x2000 + _EPOLLET = 0x80000000 + _EPOLL_CLOEXEC = 0x80000 + _EPOLL_CTL_ADD = 0x1 + _EPOLL_CTL_DEL = 0x2 + _EPOLL_CTL_MOD = 0x3 +) + +type timespec struct { + tv_sec int32 + tv_nsec int32 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = int32(x) +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = x +} + +type sigaltstackt struct { + ss_sp *byte + ss_flags int32 + ss_size uintptr +} + +type sigcontext struct { + trap_no uint32 + error_code uint32 + oldmask uint32 + r0 uint32 + r1 uint32 + r2 uint32 + r3 uint32 + r4 uint32 + r5 uint32 + r6 uint32 + r7 uint32 + r8 uint32 + r9 uint32 + r10 uint32 + fp uint32 + ip uint32 + sp uint32 + lr uint32 + pc uint32 + cpsr uint32 + fault_address uint32 +} + +type ucontext struct { + uc_flags uint32 + uc_link *ucontext + uc_stack sigaltstackt + uc_mcontext sigcontext + uc_sigmask uint32 + __unused [31]int32 + uc_regspace [128]uint32 +} + +type timeval struct { + tv_sec int32 + tv_usec int32 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + // below here is a union; si_addr is the only field we use + si_addr uint32 +} + +type sigactiont struct { + sa_handler uintptr + sa_flags uint32 + sa_restorer uintptr + sa_mask uint64 +} + +type epollevent struct { + events uint32 + _pad uint32 + data [8]byte // to match amd64 +} diff --git a/src/runtime/defs_linux_arm.h b/src/runtime/defs_linux_arm.h deleted file mode 100644 index 50b3c919e..000000000 --- a/src/runtime/defs_linux_arm.h +++ /dev/null @@ -1,168 +0,0 @@ -// TODO: Generate using cgo like defs_linux_{386,amd64}.h - -// Constants -enum { - EINTR = 0x4, - ENOMEM = 0xc, - EAGAIN = 0xb, - - PROT_NONE = 0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - MAP_ANON = 0x20, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - MADV_DONTNEED = 0x4, - SA_RESTART = 0x10000000, - SA_ONSTACK = 0x8000000, - SA_RESTORER = 0, // unused on ARM - SA_SIGINFO = 0x4, - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGBUS = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGUSR1 = 0xa, - SIGSEGV = 0xb, - SIGUSR2 = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGSTKFLT = 0x10, - SIGCHLD = 0x11, - SIGCONT = 0x12, - SIGSTOP = 0x13, - SIGTSTP = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGURG = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGIO = 0x1d, - SIGPWR = 0x1e, - SIGSYS = 0x1f, - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - ITIMER_REAL = 0, - ITIMER_PROF = 0x2, - ITIMER_VIRTUAL = 0x1, - O_RDONLY = 0, - O_CLOEXEC = 02000000, - - EPOLLIN = 0x1, - EPOLLOUT = 0x4, - EPOLLERR = 0x8, - EPOLLHUP = 0x10, - EPOLLRDHUP = 0x2000, - EPOLLET = -0x80000000, - EPOLL_CLOEXEC = 0x80000, - EPOLL_CTL_ADD = 0x1, - EPOLL_CTL_DEL = 0x2, - EPOLL_CTL_MOD = 0x3, -}; - -// Types -#pragma pack on - -typedef struct Timespec Timespec; -struct Timespec { - int32 tv_sec; - int32 tv_nsec; -}; - -typedef struct SigaltstackT SigaltstackT; -struct SigaltstackT { - void *ss_sp; - int32 ss_flags; - uint32 ss_size; -}; - -typedef struct Sigcontext Sigcontext; -struct Sigcontext { - uint32 trap_no; - uint32 error_code; - uint32 oldmask; - uint32 arm_r0; - uint32 arm_r1; - uint32 arm_r2; - uint32 arm_r3; - uint32 arm_r4; - uint32 arm_r5; - uint32 arm_r6; - uint32 arm_r7; - uint32 arm_r8; - uint32 arm_r9; - uint32 arm_r10; - uint32 arm_fp; - uint32 arm_ip; - uint32 arm_sp; - uint32 arm_lr; - uint32 arm_pc; - uint32 arm_cpsr; - uint32 fault_address; -}; - -typedef struct Ucontext Ucontext; -struct Ucontext { - uint32 uc_flags; - Ucontext *uc_link; - SigaltstackT uc_stack; - Sigcontext uc_mcontext; - uint32 uc_sigmask; - int32 __unused[31]; - uint32 uc_regspace[128]; -}; - -typedef struct Timeval Timeval; -struct Timeval { - int32 tv_sec; - int32 tv_usec; -}; - -typedef struct Itimerval Itimerval; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -typedef struct Siginfo Siginfo; -struct Siginfo { - int32 si_signo; - int32 si_errno; - int32 si_code; - uint8 _sifields[4]; -}; - -typedef struct SigactionT SigactionT; -struct SigactionT { - void *sa_handler; - uint32 sa_flags; - void *sa_restorer; - uint64 sa_mask; -}; - -typedef struct EpollEvent EpollEvent; -struct EpollEvent { - uint32 events; - uint32 _pad; - byte data[8]; // to match amd64 -}; -#pragma pack off diff --git a/src/runtime/defs_linux_power64.go b/src/runtime/defs_linux_power64.go new file mode 100644 index 000000000..f90b84874 --- /dev/null +++ b/src/runtime/defs_linux_power64.go @@ -0,0 +1,198 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs3_linux.go + +package runtime + +const ( + _EINTR = 0x4 + _EAGAIN = 0xb + _ENOMEM = 0xc + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x20 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + + _SA_RESTART = 0x10000000 + _SA_ONSTACK = 0x8000000 + _SA_SIGINFO = 0x4 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0xa + _SIGSEGV = 0xb + _SIGUSR2 = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGSTKFLT = 0x10 + _SIGCHLD = 0x11 + _SIGCONT = 0x12 + _SIGSTOP = 0x13 + _SIGTSTP = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGIO = 0x1d + _SIGPWR = 0x1e + _SIGSYS = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EPOLLIN = 0x1 + _EPOLLOUT = 0x4 + _EPOLLERR = 0x8 + _EPOLLHUP = 0x10 + _EPOLLRDHUP = 0x2000 + _EPOLLET = 0x80000000 + _EPOLL_CLOEXEC = 0x80000 + _EPOLL_CTL_ADD = 0x1 + _EPOLL_CTL_DEL = 0x2 + _EPOLL_CTL_MOD = 0x3 +) + +//struct Sigset { +// uint64 sig[1]; +//}; +//typedef uint64 Sigset; + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = int64(x) +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type sigactiont struct { + sa_handler uintptr + sa_flags uint64 + sa_restorer uintptr + sa_mask uint64 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + // below here is a union; si_addr is the only field we use + si_addr uint64 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type epollevent struct { + events uint32 + pad_cgo_0 [4]byte + data [8]byte // unaligned uintptr +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs3_linux.go + +const ( + _O_RDONLY = 0x0 + _O_CLOEXEC = 0x80000 + _SA_RESTORER = 0 +) + +type ptregs struct { + gpr [32]uint64 + nip uint64 + msr uint64 + orig_gpr3 uint64 + ctr uint64 + link uint64 + xer uint64 + ccr uint64 + softe uint64 + trap uint64 + dar uint64 + dsisr uint64 + result uint64 +} + +type vreg struct { + u [4]uint32 +} + +type sigaltstackt struct { + ss_sp *byte + ss_flags int32 + pad_cgo_0 [4]byte + ss_size uintptr +} + +type sigcontext struct { + _unused [4]uint64 + signal int32 + _pad0 int32 + handler uint64 + oldmask uint64 + regs *ptregs + gp_regs [48]uint64 + fp_regs [33]float64 + v_regs *vreg + vmx_reserve [101]int64 +} + +type ucontext struct { + uc_flags uint64 + uc_link *ucontext + uc_stack sigaltstackt + uc_sigmask uint64 + __unused [15]uint64 + uc_mcontext sigcontext +} diff --git a/src/runtime/defs_linux_power64le.go b/src/runtime/defs_linux_power64le.go new file mode 100644 index 000000000..f90b84874 --- /dev/null +++ b/src/runtime/defs_linux_power64le.go @@ -0,0 +1,198 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs3_linux.go + +package runtime + +const ( + _EINTR = 0x4 + _EAGAIN = 0xb + _ENOMEM = 0xc + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x20 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_DONTNEED = 0x4 + + _SA_RESTART = 0x10000000 + _SA_ONSTACK = 0x8000000 + _SA_SIGINFO = 0x4 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0xa + _SIGSEGV = 0xb + _SIGUSR2 = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGSTKFLT = 0x10 + _SIGCHLD = 0x11 + _SIGCONT = 0x12 + _SIGSTOP = 0x13 + _SIGTSTP = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGIO = 0x1d + _SIGPWR = 0x1e + _SIGSYS = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EPOLLIN = 0x1 + _EPOLLOUT = 0x4 + _EPOLLERR = 0x8 + _EPOLLHUP = 0x10 + _EPOLLRDHUP = 0x2000 + _EPOLLET = 0x80000000 + _EPOLL_CLOEXEC = 0x80000 + _EPOLL_CTL_ADD = 0x1 + _EPOLL_CTL_DEL = 0x2 + _EPOLL_CTL_MOD = 0x3 +) + +//struct Sigset { +// uint64 sig[1]; +//}; +//typedef uint64 Sigset; + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = int64(x) +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type sigactiont struct { + sa_handler uintptr + sa_flags uint64 + sa_restorer uintptr + sa_mask uint64 +} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + // below here is a union; si_addr is the only field we use + si_addr uint64 +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type epollevent struct { + events uint32 + pad_cgo_0 [4]byte + data [8]byte // unaligned uintptr +} + +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_linux.go defs3_linux.go + +const ( + _O_RDONLY = 0x0 + _O_CLOEXEC = 0x80000 + _SA_RESTORER = 0 +) + +type ptregs struct { + gpr [32]uint64 + nip uint64 + msr uint64 + orig_gpr3 uint64 + ctr uint64 + link uint64 + xer uint64 + ccr uint64 + softe uint64 + trap uint64 + dar uint64 + dsisr uint64 + result uint64 +} + +type vreg struct { + u [4]uint32 +} + +type sigaltstackt struct { + ss_sp *byte + ss_flags int32 + pad_cgo_0 [4]byte + ss_size uintptr +} + +type sigcontext struct { + _unused [4]uint64 + signal int32 + _pad0 int32 + handler uint64 + oldmask uint64 + regs *ptregs + gp_regs [48]uint64 + fp_regs [33]float64 + v_regs *vreg + vmx_reserve [101]int64 +} + +type ucontext struct { + uc_flags uint64 + uc_link *ucontext + uc_stack sigaltstackt + uc_sigmask uint64 + __unused [15]uint64 + uc_mcontext sigcontext +} diff --git a/src/runtime/defs_nacl_386.go b/src/runtime/defs_nacl_386.go new file mode 100644 index 000000000..498882904 --- /dev/null +++ b/src/runtime/defs_nacl_386.go @@ -0,0 +1,42 @@ +package runtime + +const ( + // These values are referred to in the source code + // but really don't matter. Even so, use the standard numbers. + _SIGSEGV = 11 + _SIGPROF = 27 +) + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +type excregs386 struct { + eax uint32 + ecx uint32 + edx uint32 + ebx uint32 + esp uint32 + ebp uint32 + esi uint32 + edi uint32 + eip uint32 + eflags uint32 +} + +type exccontext struct { + size uint32 + portable_context_offset uint32 + portable_context_size uint32 + arch uint32 + regs_size uint32 + reserved [11]uint32 + regs excregs386 +} + +type excportablecontext struct { + pc uint32 + sp uint32 + fp uint32 +} diff --git a/src/runtime/defs_nacl_386.h b/src/runtime/defs_nacl_386.h deleted file mode 100644 index e8fbb38e1..000000000 --- a/src/runtime/defs_nacl_386.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Created by hand, not machine generated. - -enum -{ - // These values are referred to in the source code - // but really don't matter. Even so, use the standard numbers. - SIGSEGV = 11, - SIGPROF = 27, -}; - -typedef struct Siginfo Siginfo; - -// native_client/src/trusted/service_runtime/include/machine/_types.h -typedef struct Timespec Timespec; - -struct Timespec -{ - int64 tv_sec; - int32 tv_nsec; -}; - -// native_client/src/trusted/service_runtime/nacl_exception.h -// native_client/src/include/nacl/nacl_exception.h - -typedef struct ExcContext ExcContext; -typedef struct ExcPortable ExcPortable; -typedef struct ExcRegs386 ExcRegs386; - -struct ExcRegs386 -{ - uint32 eax; - uint32 ecx; - uint32 edx; - uint32 ebx; - uint32 esp; - uint32 ebp; - uint32 esi; - uint32 edi; - uint32 eip; - uint32 eflags; -}; - -struct ExcContext -{ - uint32 size; - uint32 portable_context_offset; - uint32 portable_context_size; - uint32 arch; - uint32 regs_size; - uint32 reserved[11]; - ExcRegs386 regs; -}; - -struct ExcPortableContext -{ - uint32 pc; - uint32 sp; - uint32 fp; -}; diff --git a/src/runtime/defs_nacl_amd64p32.go b/src/runtime/defs_nacl_amd64p32.go new file mode 100644 index 000000000..add11fe06 --- /dev/null +++ b/src/runtime/defs_nacl_amd64p32.go @@ -0,0 +1,63 @@ +package runtime + +const ( + // These values are referred to in the source code + // but really don't matter. Even so, use the standard numbers. + _SIGSEGV = 11 + _SIGPROF = 27 +) + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +type excregs386 struct { + eax uint32 + ecx uint32 + edx uint32 + ebx uint32 + esp uint32 + ebp uint32 + esi uint32 + edi uint32 + eip uint32 + eflags uint32 +} + +type excregsamd64 struct { + rax uint64 + rcx uint64 + rdx uint64 + rbx uint64 + rsp uint64 + rbp uint64 + rsi uint64 + rdi uint64 + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + rip uint64 + rflags uint32 +} + +type exccontext struct { + size uint32 + portable_context_offset uint32 + portable_context_size uint32 + arch uint32 + regs_size uint32 + reserved [11]uint32 + regs excregsamd64 +} + +type excportablecontext struct { + pc uint32 + sp uint32 + fp uint32 +} diff --git a/src/runtime/defs_nacl_amd64p32.h b/src/runtime/defs_nacl_amd64p32.h deleted file mode 100644 index 45663d40a..000000000 --- a/src/runtime/defs_nacl_amd64p32.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Created by hand, not machine generated. - -enum -{ - // These values are referred to in the source code - // but really don't matter. Even so, use the standard numbers. - SIGSEGV = 11, - SIGPROF = 27, -}; - -typedef struct Siginfo Siginfo; - - -// native_client/src/trusted/service_runtime/include/machine/_types.h -typedef struct Timespec Timespec; - -struct Timespec -{ - int64 tv_sec; - int32 tv_nsec; -}; - -// native_client/src/trusted/service_runtime/nacl_exception.h -// native_client/src/include/nacl/nacl_exception.h - -typedef struct ExcContext ExcContext; -typedef struct ExcPortable ExcPortable; -typedef struct ExcRegs386 ExcRegs386; -typedef struct ExcRegsAmd64 ExcRegsAmd64; - -struct ExcRegs386 -{ - uint32 eax; - uint32 ecx; - uint32 edx; - uint32 ebx; - uint32 esp; - uint32 ebp; - uint32 esi; - uint32 edi; - uint32 eip; - uint32 eflags; -}; - -struct ExcRegsAmd64 -{ - uint64 rax; - uint64 rcx; - uint64 rdx; - uint64 rbx; - uint64 rsp; - uint64 rbp; - uint64 rsi; - uint64 rdi; - uint64 r8; - uint64 r9; - uint64 r10; - uint64 r11; - uint64 r12; - uint64 r13; - uint64 r14; - uint64 r15; - uint64 rip; - uint32 rflags; -}; - -struct ExcContext -{ - uint32 size; - uint32 portable_context_offset; - uint32 portable_context_size; - uint32 arch; - uint32 regs_size; - uint32 reserved[11]; - union { - ExcRegs386 regs; - ExcRegsAmd64 regs64; - } regs; -}; - -struct ExcPortableContext -{ - uint32 pc; - uint32 sp; - uint32 fp; -}; diff --git a/src/runtime/defs_nacl_arm.go b/src/runtime/defs_nacl_arm.go new file mode 100644 index 000000000..c983cffb9 --- /dev/null +++ b/src/runtime/defs_nacl_arm.go @@ -0,0 +1,49 @@ +package runtime + +const ( + // These values are referred to in the source code + // but really don't matter. Even so, use the standard numbers. + _SIGSEGV = 11 + _SIGPROF = 27 +) + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +type excregsarm struct { + r0 uint32 + r1 uint32 + r2 uint32 + r3 uint32 + r4 uint32 + r5 uint32 + r6 uint32 + r7 uint32 + r8 uint32 + r9 uint32 // the value reported here is undefined. + r10 uint32 + r11 uint32 + r12 uint32 + sp uint32 /* r13 */ + lr uint32 /* r14 */ + pc uint32 /* r15 */ + cpsr uint32 +} + +type exccontext struct { + size uint32 + portable_context_offset uint32 + portable_context_size uint32 + arch uint32 + regs_size uint32 + reserved [11]uint32 + regs excregsarm +} + +type excportablecontext struct { + pc uint32 + sp uint32 + fp uint32 +} diff --git a/src/runtime/defs_nacl_arm.h b/src/runtime/defs_nacl_arm.h deleted file mode 100644 index 9ce07ccb2..000000000 --- a/src/runtime/defs_nacl_arm.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Created by hand, not machine generated. - -enum -{ - // These values are referred to in the source code - // but really don't matter. Even so, use the standard numbers. - SIGSEGV = 11, - SIGPROF = 27, -}; - -typedef struct Siginfo Siginfo; - -// native_client/src/trusted/service_runtime/include/machine/_types.h -typedef struct Timespec Timespec; - -struct Timespec -{ - int64 tv_sec; - int32 tv_nsec; -}; - -// native_client/src/trusted/service_runtime/nacl_exception.h -// native_client/src/include/nacl/nacl_exception.h - -typedef struct ExcContext ExcContext; -typedef struct ExcPortable ExcPortable; -typedef struct ExcRegsARM ExcRegsARM; - -struct ExcRegsARM -{ - uint32 r0; - uint32 r1; - uint32 r2; - uint32 r3; - uint32 r4; - uint32 r5; - uint32 r6; - uint32 r7; - uint32 r8; - uint32 r9; // the value reported here is undefined. - uint32 r10; - uint32 r11; - uint32 r12; - uint32 sp; /* r13 */ - uint32 lr; /* r14 */ - uint32 pc; /* r15 */ - uint32 cpsr; -}; - -struct ExcContext -{ - uint32 size; - uint32 portable_context_offset; - uint32 portable_context_size; - uint32 arch; - uint32 regs_size; - uint32 reserved[11]; - ExcRegsARM regs; -}; - -struct ExcPortableContext -{ - uint32 pc; - uint32 sp; - uint32 fp; -}; diff --git a/src/runtime/defs_netbsd_386.h b/src/runtime/defs_netbsd_386.h deleted file mode 100644 index fd87804f9..000000000 --- a/src/runtime/defs_netbsd_386.h +++ /dev/null @@ -1,182 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_386.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x6, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0, - EV_ERROR = 0x4000, - EVFILT_READ = 0x0, - EVFILT_WRITE = 0x1, -}; - -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct Siginfo Siginfo; -typedef struct StackT StackT; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct McontextT McontextT; -typedef struct UcontextT UcontextT; -typedef struct KeventT KeventT; - -#pragma pack on - -struct SigaltstackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct Siginfo { - int32 _signo; - int32 _code; - int32 _errno; - byte _reason[20]; -}; - -struct StackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Timespec { - int64 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct McontextT { - int32 __gregs[19]; - byte __fpregs[644]; - int32 _mc_tlsbase; -}; -struct UcontextT { - uint32 uc_flags; - UcontextT *uc_link; - Sigset uc_sigmask; - StackT uc_stack; - McontextT uc_mcontext; - int32 __uc_pad[4]; -}; - -struct KeventT { - uint32 ident; - uint32 filter; - uint32 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_386.go - - -enum { - REG_GS = 0x0, - REG_FS = 0x1, - REG_ES = 0x2, - REG_DS = 0x3, - REG_EDI = 0x4, - REG_ESI = 0x5, - REG_EBP = 0x6, - REG_ESP = 0x7, - REG_EBX = 0x8, - REG_EDX = 0x9, - REG_ECX = 0xa, - REG_EAX = 0xb, - REG_TRAPNO = 0xc, - REG_ERR = 0xd, - REG_EIP = 0xe, - REG_CS = 0xf, - REG_EFL = 0x10, - REG_UESP = 0x11, - REG_SS = 0x12, -}; - diff --git a/src/runtime/defs_netbsd_amd64.h b/src/runtime/defs_netbsd_amd64.h deleted file mode 100644 index dac94b113..000000000 --- a/src/runtime/defs_netbsd_amd64.h +++ /dev/null @@ -1,194 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x6, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0, - EV_ERROR = 0x4000, - EVFILT_READ = 0x0, - EVFILT_WRITE = 0x1, -}; - -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct Siginfo Siginfo; -typedef struct StackT StackT; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct McontextT McontextT; -typedef struct UcontextT UcontextT; -typedef struct KeventT KeventT; - -#pragma pack on - -struct SigaltstackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct Siginfo { - int32 _signo; - int32 _code; - int32 _errno; - int32 _pad; - byte _reason[24]; -}; - -struct StackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; - byte Pad_cgo_0[4]; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct McontextT { - uint64 __gregs[26]; - uint64 _mc_tlsbase; - int8 __fpregs[512]; -}; -struct UcontextT { - uint32 uc_flags; - byte Pad_cgo_0[4]; - UcontextT *uc_link; - Sigset uc_sigmask; - StackT uc_stack; - McontextT uc_mcontext; -}; - -struct KeventT { - uint64 ident; - uint32 filter; - uint32 flags; - uint32 fflags; - byte Pad_cgo_0[4]; - int64 data; - byte *udata; -}; - - -#pragma pack off -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go - - -enum { - REG_RDI = 0x0, - REG_RSI = 0x1, - REG_RDX = 0x2, - REG_RCX = 0x3, - REG_R8 = 0x4, - REG_R9 = 0x5, - REG_R10 = 0x6, - REG_R11 = 0x7, - REG_R12 = 0x8, - REG_R13 = 0x9, - REG_R14 = 0xa, - REG_R15 = 0xb, - REG_RBP = 0xc, - REG_RBX = 0xd, - REG_RAX = 0xe, - REG_GS = 0xf, - REG_FS = 0x10, - REG_ES = 0x11, - REG_DS = 0x12, - REG_TRAPNO = 0x13, - REG_ERR = 0x14, - REG_RIP = 0x15, - REG_CS = 0x16, - REG_RFLAGS = 0x17, - REG_RSP = 0x18, - REG_SS = 0x19, -}; - diff --git a/src/runtime/defs_netbsd_arm.h b/src/runtime/defs_netbsd_arm.h deleted file mode 100644 index 70f34af47..000000000 --- a/src/runtime/defs_netbsd_arm.h +++ /dev/null @@ -1,184 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x6, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_RECEIPT = 0, - EV_ERROR = 0x4000, - EVFILT_READ = 0x0, - EVFILT_WRITE = 0x1, -}; - -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct Siginfo Siginfo; -typedef struct StackT StackT; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct McontextT McontextT; -typedef struct UcontextT UcontextT; -typedef struct KeventT KeventT; - -#pragma pack on - -struct SigaltstackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigset { - uint32 __bits[4]; -}; -struct Siginfo { - int32 _signo; - int32 _code; - int32 _errno; - byte _reason[20]; -}; - -struct StackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Timespec { - int64 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct McontextT { - uint32 __gregs[17]; -#ifdef __ARM_EABI__ - byte __fpu[4+8*32+4]; -#else - byte __fpu[4+4*33+4]; -#endif - uint32 _mc_tlsbase; -}; -struct UcontextT { - uint32 uc_flags; - UcontextT *uc_link; - Sigset uc_sigmask; - StackT uc_stack; - McontextT uc_mcontext; - int32 __uc_pad[2]; -}; - -struct KeventT { - uint32 ident; - uint32 filter; - uint32 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go - - -enum { - REG_R0 = 0x0, - REG_R1 = 0x1, - REG_R2 = 0x2, - REG_R3 = 0x3, - REG_R4 = 0x4, - REG_R5 = 0x5, - REG_R6 = 0x6, - REG_R7 = 0x7, - REG_R8 = 0x8, - REG_R9 = 0x9, - REG_R10 = 0xa, - REG_R11 = 0xb, - REG_R12 = 0xc, - REG_R13 = 0xd, - REG_R14 = 0xe, - REG_R15 = 0xf, - REG_CPSR = 0x10, -}; - diff --git a/src/runtime/defs_openbsd_386.go b/src/runtime/defs_openbsd_386.go new file mode 100644 index 000000000..4b6015811 --- /dev/null +++ b/src/runtime/defs_openbsd_386.go @@ -0,0 +1,170 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_openbsd.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x6 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type tforkt struct { + tf_tcb unsafe.Pointer + tf_tid *int32 + tf_stack uintptr +} + +type sigaltstackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 +} + +type sigcontext struct { + sc_gs uint32 + sc_fs uint32 + sc_es uint32 + sc_ds uint32 + sc_edi uint32 + sc_esi uint32 + sc_ebp uint32 + sc_ebx uint32 + sc_edx uint32 + sc_ecx uint32 + sc_eax uint32 + sc_eip uint32 + sc_cs uint32 + sc_eflags uint32 + sc_esp uint32 + sc_ss uint32 + __sc_unused uint32 + sc_mask uint32 + sc_trapno uint32 + sc_err uint32 + sc_fpstate unsafe.Pointer +} + +type siginfo struct { + si_signo int32 + si_code int32 + si_errno int32 + _data [116]byte +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int32 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = x +} + +type timeval struct { + tv_sec int64 + tv_usec int32 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint32 + filter int16 + flags uint16 + fflags uint32 + data int64 + udata *byte +} diff --git a/src/runtime/defs_openbsd_386.h b/src/runtime/defs_openbsd_386.h deleted file mode 100644 index 6b77e0084..000000000 --- a/src/runtime/defs_openbsd_386.h +++ /dev/null @@ -1,168 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_openbsd.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x6, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct TforkT TforkT; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigcontext Sigcontext; -typedef struct Siginfo Siginfo; -typedef struct StackT StackT; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct TforkT { - byte *tf_tcb; - int32 *tf_tid; - byte *tf_stack; -}; - -struct SigaltstackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; -struct Sigcontext { - int32 sc_gs; - int32 sc_fs; - int32 sc_es; - int32 sc_ds; - int32 sc_edi; - int32 sc_esi; - int32 sc_ebp; - int32 sc_ebx; - int32 sc_edx; - int32 sc_ecx; - int32 sc_eax; - int32 sc_eip; - int32 sc_cs; - int32 sc_eflags; - int32 sc_esp; - int32 sc_ss; - int32 __sc_unused; - int32 sc_mask; - int32 sc_trapno; - int32 sc_err; - void *sc_fpstate; -}; -struct Siginfo { - int32 si_signo; - int32 si_code; - int32 si_errno; - byte _data[116]; -}; -typedef uint32 Sigset; -typedef byte Sigval[4]; - -struct StackT { - byte *ss_sp; - uint32 ss_size; - int32 ss_flags; -}; - -struct Timespec { - int64 tv_sec; - int32 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int32 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint32 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_openbsd_amd64.go b/src/runtime/defs_openbsd_amd64.go new file mode 100644 index 000000000..3c27c9144 --- /dev/null +++ b/src/runtime/defs_openbsd_amd64.go @@ -0,0 +1,181 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_openbsd.go + +package runtime + +import "unsafe" + +const ( + _EINTR = 0x4 + _EFAULT = 0xe + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANON = 0x1000 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x10 + + _MADV_FREE = 0x6 + + _SA_SIGINFO = 0x40 + _SA_RESTART = 0x2 + _SA_ONSTACK = 0x1 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGEMT = 0x7 + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGBUS = 0xa + _SIGSEGV = 0xb + _SIGSYS = 0xc + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGTERM = 0xf + _SIGURG = 0x10 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGCONT = 0x13 + _SIGCHLD = 0x14 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGIO = 0x17 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x1a + _SIGPROF = 0x1b + _SIGWINCH = 0x1c + _SIGINFO = 0x1d + _SIGUSR1 = 0x1e + _SIGUSR2 = 0x1f + + _FPE_INTDIV = 0x1 + _FPE_INTOVF = 0x2 + _FPE_FLTDIV = 0x3 + _FPE_FLTOVF = 0x4 + _FPE_FLTUND = 0x5 + _FPE_FLTRES = 0x6 + _FPE_FLTINV = 0x7 + _FPE_FLTSUB = 0x8 + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + + _SEGV_MAPERR = 0x1 + _SEGV_ACCERR = 0x2 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _EV_ADD = 0x1 + _EV_DELETE = 0x2 + _EV_CLEAR = 0x20 + _EV_ERROR = 0x4000 + _EVFILT_READ = -0x1 + _EVFILT_WRITE = -0x2 +) + +type tforkt struct { + tf_tcb unsafe.Pointer + tf_tid *int32 + tf_stack uintptr +} + +type sigaltstackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type sigcontext struct { + sc_rdi uint64 + sc_rsi uint64 + sc_rdx uint64 + sc_rcx uint64 + sc_r8 uint64 + sc_r9 uint64 + sc_r10 uint64 + sc_r11 uint64 + sc_r12 uint64 + sc_r13 uint64 + sc_r14 uint64 + sc_r15 uint64 + sc_rbp uint64 + sc_rbx uint64 + sc_rax uint64 + sc_gs uint64 + sc_fs uint64 + sc_es uint64 + sc_ds uint64 + sc_trapno uint64 + sc_err uint64 + sc_rip uint64 + sc_cs uint64 + sc_rflags uint64 + sc_rsp uint64 + sc_ss uint64 + sc_fpstate unsafe.Pointer + __sc_unused int32 + sc_mask int32 +} + +type siginfo struct { + si_signo int32 + si_code int32 + si_errno int32 + pad_cgo_0 [4]byte + _data [120]byte +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + pad_cgo_0 [4]byte +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +func (ts *timespec) set_sec(x int64) { + ts.tv_sec = x +} + +func (ts *timespec) set_nsec(x int32) { + ts.tv_nsec = int64(x) +} + +type timeval struct { + tv_sec int64 + tv_usec int64 +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = int64(x) +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type keventt struct { + ident uint64 + filter int16 + flags uint16 + fflags uint32 + data int64 + udata *byte +} diff --git a/src/runtime/defs_openbsd_amd64.h b/src/runtime/defs_openbsd_amd64.h deleted file mode 100644 index 761e8e47d..000000000 --- a/src/runtime/defs_openbsd_amd64.h +++ /dev/null @@ -1,179 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_openbsd.go - - -enum { - EINTR = 0x4, - EFAULT = 0xe, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x1000, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x6, - - SA_SIGINFO = 0x40, - SA_RESTART = 0x2, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x10, - SIGSTOP = 0x11, - SIGTSTP = 0x12, - SIGCONT = 0x13, - SIGCHLD = 0x14, - SIGTTIN = 0x15, - SIGTTOU = 0x16, - SIGIO = 0x17, - SIGXCPU = 0x18, - SIGXFSZ = 0x19, - SIGVTALRM = 0x1a, - SIGPROF = 0x1b, - SIGWINCH = 0x1c, - SIGINFO = 0x1d, - SIGUSR1 = 0x1e, - SIGUSR2 = 0x1f, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - EV_ADD = 0x1, - EV_DELETE = 0x2, - EV_CLEAR = 0x20, - EV_ERROR = 0x4000, - EVFILT_READ = -0x1, - EVFILT_WRITE = -0x2, -}; - -typedef struct TforkT TforkT; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigcontext Sigcontext; -typedef struct Siginfo Siginfo; -typedef struct StackT StackT; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct KeventT KeventT; - -#pragma pack on - -struct TforkT { - byte *tf_tcb; - int32 *tf_tid; - byte *tf_stack; -}; - -struct SigaltstackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -struct Sigcontext { - int64 sc_rdi; - int64 sc_rsi; - int64 sc_rdx; - int64 sc_rcx; - int64 sc_r8; - int64 sc_r9; - int64 sc_r10; - int64 sc_r11; - int64 sc_r12; - int64 sc_r13; - int64 sc_r14; - int64 sc_r15; - int64 sc_rbp; - int64 sc_rbx; - int64 sc_rax; - int64 sc_gs; - int64 sc_fs; - int64 sc_es; - int64 sc_ds; - int64 sc_trapno; - int64 sc_err; - int64 sc_rip; - int64 sc_cs; - int64 sc_rflags; - int64 sc_rsp; - int64 sc_ss; - void *sc_fpstate; - int32 __sc_unused; - int32 sc_mask; -}; -struct Siginfo { - int32 si_signo; - int32 si_code; - int32 si_errno; - byte Pad_cgo_0[4]; - byte _data[120]; -}; -typedef uint32 Sigset; -typedef byte Sigval[8]; - -struct StackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int64 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct KeventT { - uint64 ident; - int16 filter; - uint16 flags; - uint32 fflags; - int64 data; - byte *udata; -}; - - -#pragma pack off diff --git a/src/runtime/defs_plan9_386.go b/src/runtime/defs_plan9_386.go new file mode 100644 index 000000000..170506b23 --- /dev/null +++ b/src/runtime/defs_plan9_386.go @@ -0,0 +1,23 @@ +package runtime + +type ureg struct { + di uint32 /* general registers */ + si uint32 /* ... */ + bp uint32 /* ... */ + nsp uint32 + bx uint32 /* ... */ + dx uint32 /* ... */ + cx uint32 /* ... */ + ax uint32 /* ... */ + gs uint32 /* data segments */ + fs uint32 /* ... */ + es uint32 /* ... */ + ds uint32 /* ... */ + trap uint32 /* trap _type */ + ecode uint32 /* error code (or zero) */ + pc uint32 /* pc */ + cs uint32 /* old context */ + flags uint32 /* old flags */ + sp uint32 + ss uint32 /* old stack segment */ +} diff --git a/src/runtime/defs_plan9_386.h b/src/runtime/defs_plan9_386.h deleted file mode 100644 index a762b8589..000000000 --- a/src/runtime/defs_plan9_386.h +++ /dev/null @@ -1,26 +0,0 @@ -#define PAGESIZE 0x1000 - -typedef struct Ureg Ureg; - -struct Ureg -{ - uint32 di; /* general registers */ - uint32 si; /* ... */ - uint32 bp; /* ... */ - uint32 nsp; - uint32 bx; /* ... */ - uint32 dx; /* ... */ - uint32 cx; /* ... */ - uint32 ax; /* ... */ - uint32 gs; /* data segments */ - uint32 fs; /* ... */ - uint32 es; /* ... */ - uint32 ds; /* ... */ - uint32 trap; /* trap type */ - uint32 ecode; /* error code (or zero) */ - uint32 pc; /* pc */ - uint32 cs; /* old context */ - uint32 flags; /* old flags */ - uint32 sp; - uint32 ss; /* old stack segment */ -}; diff --git a/src/runtime/defs_plan9_amd64.go b/src/runtime/defs_plan9_amd64.go new file mode 100644 index 000000000..17becfb66 --- /dev/null +++ b/src/runtime/defs_plan9_amd64.go @@ -0,0 +1,32 @@ +package runtime + +type ureg struct { + ax uint64 + bx uint64 + cx uint64 + dx uint64 + si uint64 + di uint64 + bp uint64 + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + + ds uint16 + es uint16 + fs uint16 + gs uint16 + + _type uint64 + error uint64 /* error code (or zero) */ + ip uint64 /* pc */ + cs uint64 /* old context */ + flags uint64 /* old flags */ + sp uint64 /* sp */ + ss uint64 /* old stack segment */ +} diff --git a/src/runtime/defs_plan9_amd64.h b/src/runtime/defs_plan9_amd64.h deleted file mode 100644 index 20bca479c..000000000 --- a/src/runtime/defs_plan9_amd64.h +++ /dev/null @@ -1,34 +0,0 @@ -#define PAGESIZE 0x1000 - -typedef struct Ureg Ureg; - -struct Ureg { - uint64 ax; - uint64 bx; - uint64 cx; - uint64 dx; - uint64 si; - uint64 di; - uint64 bp; - uint64 r8; - uint64 r9; - uint64 r10; - uint64 r11; - uint64 r12; - uint64 r13; - uint64 r14; - uint64 r15; - - uint16 ds; - uint16 es; - uint16 fs; - uint16 gs; - - uint64 type; - uint64 error; /* error code (or zero) */ - uint64 ip; /* pc */ - uint64 cs; /* old context */ - uint64 flags; /* old flags */ - uint64 sp; /* sp */ - uint64 ss; /* old stack segment */ -}; diff --git a/src/runtime/defs_solaris_amd64.h b/src/runtime/defs_solaris_amd64.h deleted file mode 100644 index cb1cfeadc..000000000 --- a/src/runtime/defs_solaris_amd64.h +++ /dev/null @@ -1,254 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_solaris.go defs_solaris_amd64.go - - -enum { - EINTR = 0x4, - EBADF = 0x9, - EFAULT = 0xe, - EAGAIN = 0xb, - ETIMEDOUT = 0x91, - EWOULDBLOCK = 0xb, - EINPROGRESS = 0x96, - - PROT_NONE = 0x0, - PROT_READ = 0x1, - PROT_WRITE = 0x2, - PROT_EXEC = 0x4, - - MAP_ANON = 0x100, - MAP_PRIVATE = 0x2, - MAP_FIXED = 0x10, - - MADV_FREE = 0x5, - - SA_SIGINFO = 0x8, - SA_RESTART = 0x4, - SA_ONSTACK = 0x1, - - SIGHUP = 0x1, - SIGINT = 0x2, - SIGQUIT = 0x3, - SIGILL = 0x4, - SIGTRAP = 0x5, - SIGABRT = 0x6, - SIGEMT = 0x7, - SIGFPE = 0x8, - SIGKILL = 0x9, - SIGBUS = 0xa, - SIGSEGV = 0xb, - SIGSYS = 0xc, - SIGPIPE = 0xd, - SIGALRM = 0xe, - SIGTERM = 0xf, - SIGURG = 0x15, - SIGSTOP = 0x17, - SIGTSTP = 0x18, - SIGCONT = 0x19, - SIGCHLD = 0x12, - SIGTTIN = 0x1a, - SIGTTOU = 0x1b, - SIGIO = 0x16, - SIGXCPU = 0x1e, - SIGXFSZ = 0x1f, - SIGVTALRM = 0x1c, - SIGPROF = 0x1d, - SIGWINCH = 0x14, - SIGUSR1 = 0x10, - SIGUSR2 = 0x11, - - FPE_INTDIV = 0x1, - FPE_INTOVF = 0x2, - FPE_FLTDIV = 0x3, - FPE_FLTOVF = 0x4, - FPE_FLTUND = 0x5, - FPE_FLTRES = 0x6, - FPE_FLTINV = 0x7, - FPE_FLTSUB = 0x8, - - BUS_ADRALN = 0x1, - BUS_ADRERR = 0x2, - BUS_OBJERR = 0x3, - - SEGV_MAPERR = 0x1, - SEGV_ACCERR = 0x2, - - ITIMER_REAL = 0x0, - ITIMER_VIRTUAL = 0x1, - ITIMER_PROF = 0x2, - - _SC_NPROCESSORS_ONLN = 0xf, - - PTHREAD_CREATE_DETACHED = 0x40, - - FORK_NOSIGCHLD = 0x1, - FORK_WAITPID = 0x2, - - MAXHOSTNAMELEN = 0x100, - - O_NONBLOCK = 0x80, - FD_CLOEXEC = 0x1, - F_GETFL = 0x3, - F_SETFL = 0x4, - F_SETFD = 0x2, - - POLLIN = 0x1, - POLLOUT = 0x4, - POLLHUP = 0x10, - POLLERR = 0x8, - - PORT_SOURCE_FD = 0x4, -}; - -typedef struct SemT SemT; -typedef struct SigaltstackT SigaltstackT; -typedef struct Sigset Sigset; -typedef struct StackT StackT; -typedef struct Siginfo Siginfo; -typedef struct SigactionT SigactionT; -typedef struct Fpregset Fpregset; -typedef struct Mcontext Mcontext; -typedef struct Ucontext Ucontext; -typedef struct Timespec Timespec; -typedef struct Timeval Timeval; -typedef struct Itimerval Itimerval; -typedef struct PortEvent PortEvent; -typedef struct PthreadAttr PthreadAttr; -typedef struct Stat Stat; - -#pragma pack on - -struct SemT { - uint32 sem_count; - uint16 sem_type; - uint16 sem_magic; - uint64 sem_pad1[3]; - uint64 sem_pad2[2]; -}; - -struct SigaltstackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; -struct Sigset { - uint32 __sigbits[4]; -}; -struct StackT { - byte *ss_sp; - uint64 ss_size; - int32 ss_flags; - byte Pad_cgo_0[4]; -}; - -struct Siginfo { - int32 si_signo; - int32 si_code; - int32 si_errno; - int32 si_pad; - byte __data[240]; -}; -struct SigactionT { - int32 sa_flags; - byte Pad_cgo_0[4]; - byte _funcptr[8]; - Sigset sa_mask; -}; - -struct Fpregset { - byte fp_reg_set[528]; -}; -struct Mcontext { - int64 gregs[28]; - Fpregset fpregs; -}; -struct Ucontext { - uint64 uc_flags; - Ucontext *uc_link; - Sigset uc_sigmask; - StackT uc_stack; - byte Pad_cgo_0[8]; - Mcontext uc_mcontext; - int64 uc_filler[5]; - byte Pad_cgo_1[8]; -}; - -struct Timespec { - int64 tv_sec; - int64 tv_nsec; -}; -struct Timeval { - int64 tv_sec; - int64 tv_usec; -}; -struct Itimerval { - Timeval it_interval; - Timeval it_value; -}; - -struct PortEvent { - int32 portev_events; - uint16 portev_source; - uint16 portev_pad; - uint64 portev_object; - byte *portev_user; -}; -typedef uint32 Pthread; -struct PthreadAttr { - byte *__pthread_attrp; -}; - -struct Stat { - uint64 st_dev; - uint64 st_ino; - uint32 st_mode; - uint32 st_nlink; - uint32 st_uid; - uint32 st_gid; - uint64 st_rdev; - int64 st_size; - Timespec st_atim; - Timespec st_mtim; - Timespec st_ctim; - int32 st_blksize; - byte Pad_cgo_0[4]; - int64 st_blocks; - int8 st_fstype[16]; -}; - - -#pragma pack off -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_solaris.go defs_solaris_amd64.go - - -enum { - REG_RDI = 0x8, - REG_RSI = 0x9, - REG_RDX = 0xc, - REG_RCX = 0xd, - REG_R8 = 0x7, - REG_R9 = 0x6, - REG_R10 = 0x5, - REG_R11 = 0x4, - REG_R12 = 0x3, - REG_R13 = 0x2, - REG_R14 = 0x1, - REG_R15 = 0x0, - REG_RBP = 0xa, - REG_RBX = 0xb, - REG_RAX = 0xe, - REG_GS = 0x17, - REG_FS = 0x16, - REG_ES = 0x18, - REG_DS = 0x19, - REG_TRAPNO = 0xf, - REG_ERR = 0x10, - REG_RIP = 0x11, - REG_CS = 0x12, - REG_RFLAGS = 0x13, - REG_RSP = 0x14, - REG_SS = 0x15, -}; - diff --git a/src/runtime/defs_windows_386.go b/src/runtime/defs_windows_386.go new file mode 100644 index 000000000..abec2d839 --- /dev/null +++ b/src/runtime/defs_windows_386.go @@ -0,0 +1,109 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_windows.go + +package runtime + +const ( + _PROT_NONE = 0 + _PROT_READ = 1 + _PROT_WRITE = 2 + _PROT_EXEC = 4 + + _MAP_ANON = 1 + _MAP_PRIVATE = 2 + + _DUPLICATE_SAME_ACCESS = 0x2 + _THREAD_PRIORITY_HIGHEST = 0x2 + + _SIGINT = 0x2 + _CTRL_C_EVENT = 0x0 + _CTRL_BREAK_EVENT = 0x1 + + _CONTEXT_CONTROL = 0x10001 + _CONTEXT_FULL = 0x10007 + + _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 + _EXCEPTION_BREAKPOINT = 0x80000003 + _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d + _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e + _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f + _EXCEPTION_FLT_OVERFLOW = 0xc0000091 + _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 + _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 + _EXCEPTION_INT_OVERFLOW = 0xc0000095 + + _INFINITE = 0xffffffff + _WAIT_TIMEOUT = 0x102 + + _EXCEPTION_CONTINUE_EXECUTION = -0x1 + _EXCEPTION_CONTINUE_SEARCH = 0x0 +) + +type systeminfo struct { + anon0 [4]byte + dwpagesize uint32 + lpminimumapplicationaddress *byte + lpmaximumapplicationaddress *byte + dwactiveprocessormask uint32 + dwnumberofprocessors uint32 + dwprocessortype uint32 + dwallocationgranularity uint32 + wprocessorlevel uint16 + wprocessorrevision uint16 +} + +type exceptionrecord struct { + exceptioncode uint32 + exceptionflags uint32 + exceptionrecord *exceptionrecord + exceptionaddress *byte + numberparameters uint32 + exceptioninformation [15]uint32 +} + +type floatingsavearea struct { + controlword uint32 + statusword uint32 + tagword uint32 + erroroffset uint32 + errorselector uint32 + dataoffset uint32 + dataselector uint32 + registerarea [80]uint8 + cr0npxstate uint32 +} + +type context struct { + contextflags uint32 + dr0 uint32 + dr1 uint32 + dr2 uint32 + dr3 uint32 + dr6 uint32 + dr7 uint32 + floatsave floatingsavearea + seggs uint32 + segfs uint32 + seges uint32 + segds uint32 + edi uint32 + esi uint32 + ebx uint32 + edx uint32 + ecx uint32 + eax uint32 + ebp uint32 + eip uint32 + segcs uint32 + eflags uint32 + esp uint32 + segss uint32 + extendedregisters [512]uint8 +} + +type overlapped struct { + internal uint32 + internalhigh uint32 + anon0 [8]byte + hevent *byte +} diff --git a/src/runtime/defs_windows_386.h b/src/runtime/defs_windows_386.h deleted file mode 100644 index 2317c04f6..000000000 --- a/src/runtime/defs_windows_386.h +++ /dev/null @@ -1,116 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_windows.go - - -enum { - PROT_NONE = 0, - PROT_READ = 1, - PROT_WRITE = 2, - PROT_EXEC = 4, - - MAP_ANON = 1, - MAP_PRIVATE = 2, - - DUPLICATE_SAME_ACCESS = 0x2, - THREAD_PRIORITY_HIGHEST = 0x2, - - SIGINT = 0x2, - CTRL_C_EVENT = 0x0, - CTRL_BREAK_EVENT = 0x1, - - CONTEXT_CONTROL = 0x10001, - CONTEXT_FULL = 0x10007, - - EXCEPTION_ACCESS_VIOLATION = 0xc0000005, - EXCEPTION_BREAKPOINT = 0x80000003, - EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d, - EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e, - EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f, - EXCEPTION_FLT_OVERFLOW = 0xc0000091, - EXCEPTION_FLT_UNDERFLOW = 0xc0000093, - EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094, - EXCEPTION_INT_OVERFLOW = 0xc0000095, - - INFINITE = 0xffffffff, - WAIT_TIMEOUT = 0x102, - - EXCEPTION_CONTINUE_EXECUTION = -0x1, - EXCEPTION_CONTINUE_SEARCH = 0x0, -}; - -typedef struct SystemInfo SystemInfo; -typedef struct ExceptionRecord ExceptionRecord; -typedef struct FloatingSaveArea FloatingSaveArea; -typedef struct M128a M128a; -typedef struct Context Context; -typedef struct Overlapped Overlapped; - -#pragma pack on - -struct SystemInfo { - byte anon0[4]; - uint32 dwPageSize; - byte *lpMinimumApplicationAddress; - byte *lpMaximumApplicationAddress; - uint32 dwActiveProcessorMask; - uint32 dwNumberOfProcessors; - uint32 dwProcessorType; - uint32 dwAllocationGranularity; - uint16 wProcessorLevel; - uint16 wProcessorRevision; -}; -struct ExceptionRecord { - uint32 ExceptionCode; - uint32 ExceptionFlags; - ExceptionRecord *ExceptionRecord; - byte *ExceptionAddress; - uint32 NumberParameters; - uint32 ExceptionInformation[15]; -}; -struct FloatingSaveArea { - uint32 ControlWord; - uint32 StatusWord; - uint32 TagWord; - uint32 ErrorOffset; - uint32 ErrorSelector; - uint32 DataOffset; - uint32 DataSelector; - uint8 RegisterArea[80]; - uint32 Cr0NpxState; -}; -struct Context { - uint32 ContextFlags; - uint32 Dr0; - uint32 Dr1; - uint32 Dr2; - uint32 Dr3; - uint32 Dr6; - uint32 Dr7; - FloatingSaveArea FloatSave; - uint32 SegGs; - uint32 SegFs; - uint32 SegEs; - uint32 SegDs; - uint32 Edi; - uint32 Esi; - uint32 Ebx; - uint32 Edx; - uint32 Ecx; - uint32 Eax; - uint32 Ebp; - uint32 Eip; - uint32 SegCs; - uint32 EFlags; - uint32 Esp; - uint32 SegSs; - uint8 ExtendedRegisters[512]; -}; -struct Overlapped { - uint32 Internal; - uint32 InternalHigh; - byte anon0[8]; - byte *hEvent; -}; - - -#pragma pack off diff --git a/src/runtime/defs_windows_amd64.go b/src/runtime/defs_windows_amd64.go new file mode 100644 index 000000000..81b13597b --- /dev/null +++ b/src/runtime/defs_windows_amd64.go @@ -0,0 +1,124 @@ +// created by cgo -cdefs and then converted to Go +// cgo -cdefs defs_windows.go + +package runtime + +const ( + _PROT_NONE = 0 + _PROT_READ = 1 + _PROT_WRITE = 2 + _PROT_EXEC = 4 + + _MAP_ANON = 1 + _MAP_PRIVATE = 2 + + _DUPLICATE_SAME_ACCESS = 0x2 + _THREAD_PRIORITY_HIGHEST = 0x2 + + _SIGINT = 0x2 + _CTRL_C_EVENT = 0x0 + _CTRL_BREAK_EVENT = 0x1 + + _CONTEXT_CONTROL = 0x100001 + _CONTEXT_FULL = 0x10000b + + _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 + _EXCEPTION_BREAKPOINT = 0x80000003 + _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d + _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e + _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f + _EXCEPTION_FLT_OVERFLOW = 0xc0000091 + _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 + _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 + _EXCEPTION_INT_OVERFLOW = 0xc0000095 + + _INFINITE = 0xffffffff + _WAIT_TIMEOUT = 0x102 + + _EXCEPTION_CONTINUE_EXECUTION = -0x1 + _EXCEPTION_CONTINUE_SEARCH = 0x0 +) + +type systeminfo struct { + anon0 [4]byte + dwpagesize uint32 + lpminimumapplicationaddress *byte + lpmaximumapplicationaddress *byte + dwactiveprocessormask uint64 + dwnumberofprocessors uint32 + dwprocessortype uint32 + dwallocationgranularity uint32 + wprocessorlevel uint16 + wprocessorrevision uint16 +} + +type exceptionrecord struct { + exceptioncode uint32 + exceptionflags uint32 + exceptionrecord *exceptionrecord + exceptionaddress *byte + numberparameters uint32 + pad_cgo_0 [4]byte + exceptioninformation [15]uint64 +} + +type m128a struct { + low uint64 + high int64 +} + +type context struct { + p1home uint64 + p2home uint64 + p3home uint64 + p4home uint64 + p5home uint64 + p6home uint64 + contextflags uint32 + mxcsr uint32 + segcs uint16 + segds uint16 + seges uint16 + segfs uint16 + seggs uint16 + segss uint16 + eflags uint32 + dr0 uint64 + dr1 uint64 + dr2 uint64 + dr3 uint64 + dr6 uint64 + dr7 uint64 + rax uint64 + rcx uint64 + rdx uint64 + rbx uint64 + rsp uint64 + rbp uint64 + rsi uint64 + rdi uint64 + r8 uint64 + r9 uint64 + r10 uint64 + r11 uint64 + r12 uint64 + r13 uint64 + r14 uint64 + r15 uint64 + rip uint64 + anon0 [512]byte + vectorregister [26]m128a + vectorcontrol uint64 + debugcontrol uint64 + lastbranchtorip uint64 + lastbranchfromrip uint64 + lastexceptiontorip uint64 + lastexceptionfromrip uint64 +} + +type overlapped struct { + internal uint64 + internalhigh uint64 + anon0 [8]byte + hevent *byte +} diff --git a/src/runtime/defs_windows_amd64.h b/src/runtime/defs_windows_amd64.h deleted file mode 100644 index 7f37a7a8c..000000000 --- a/src/runtime/defs_windows_amd64.h +++ /dev/null @@ -1,131 +0,0 @@ -// Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_windows.go - - -enum { - PROT_NONE = 0, - PROT_READ = 1, - PROT_WRITE = 2, - PROT_EXEC = 4, - - MAP_ANON = 1, - MAP_PRIVATE = 2, - - DUPLICATE_SAME_ACCESS = 0x2, - THREAD_PRIORITY_HIGHEST = 0x2, - - SIGINT = 0x2, - CTRL_C_EVENT = 0x0, - CTRL_BREAK_EVENT = 0x1, - - CONTEXT_CONTROL = 0x100001, - CONTEXT_FULL = 0x10000b, - - EXCEPTION_ACCESS_VIOLATION = 0xc0000005, - EXCEPTION_BREAKPOINT = 0x80000003, - EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d, - EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e, - EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f, - EXCEPTION_FLT_OVERFLOW = 0xc0000091, - EXCEPTION_FLT_UNDERFLOW = 0xc0000093, - EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094, - EXCEPTION_INT_OVERFLOW = 0xc0000095, - - INFINITE = 0xffffffff, - WAIT_TIMEOUT = 0x102, - - EXCEPTION_CONTINUE_EXECUTION = -0x1, - EXCEPTION_CONTINUE_SEARCH = 0x0, -}; - -typedef struct SystemInfo SystemInfo; -typedef struct ExceptionRecord ExceptionRecord; -typedef struct FloatingSaveArea FloatingSaveArea; -typedef struct M128a M128a; -typedef struct Context Context; -typedef struct Overlapped Overlapped; - -#pragma pack on - -struct SystemInfo { - byte anon0[4]; - uint32 dwPageSize; - byte *lpMinimumApplicationAddress; - byte *lpMaximumApplicationAddress; - uint64 dwActiveProcessorMask; - uint32 dwNumberOfProcessors; - uint32 dwProcessorType; - uint32 dwAllocationGranularity; - uint16 wProcessorLevel; - uint16 wProcessorRevision; -}; -struct ExceptionRecord { - uint32 ExceptionCode; - uint32 ExceptionFlags; - ExceptionRecord *ExceptionRecord; - byte *ExceptionAddress; - uint32 NumberParameters; - byte Pad_cgo_0[4]; - uint64 ExceptionInformation[15]; -}; -struct M128a { - uint64 Low; - int64 High; -}; -struct Context { - uint64 P1Home; - uint64 P2Home; - uint64 P3Home; - uint64 P4Home; - uint64 P5Home; - uint64 P6Home; - uint32 ContextFlags; - uint32 MxCsr; - uint16 SegCs; - uint16 SegDs; - uint16 SegEs; - uint16 SegFs; - uint16 SegGs; - uint16 SegSs; - uint32 EFlags; - uint64 Dr0; - uint64 Dr1; - uint64 Dr2; - uint64 Dr3; - uint64 Dr6; - uint64 Dr7; - uint64 Rax; - uint64 Rcx; - uint64 Rdx; - uint64 Rbx; - uint64 Rsp; - uint64 Rbp; - uint64 Rsi; - uint64 Rdi; - uint64 R8; - uint64 R9; - uint64 R10; - uint64 R11; - uint64 R12; - uint64 R13; - uint64 R14; - uint64 R15; - uint64 Rip; - byte anon0[512]; - M128a VectorRegister[26]; - uint64 VectorControl; - uint64 DebugControl; - uint64 LastBranchToRip; - uint64 LastBranchFromRip; - uint64 LastExceptionToRip; - uint64 LastExceptionFromRip; -}; -struct Overlapped { - uint64 Internal; - uint64 InternalHigh; - byte anon0[8]; - byte *hEvent; -}; - - -#pragma pack off diff --git a/src/runtime/env_posix.go b/src/runtime/env_posix.go index dd57872d7..03c7a5a4a 100644 --- a/src/runtime/env_posix.go +++ b/src/runtime/env_posix.go @@ -8,8 +8,6 @@ package runtime import "unsafe" -func environ() []string - func getenv(s *byte) *byte { val := gogetenv(gostringnocopy(s)) if val == "" { @@ -32,13 +30,13 @@ func gogetenv(key string) string { return "" } -var _cgo_setenv uintptr // pointer to C function -var _cgo_unsetenv uintptr // pointer to C function +var _cgo_setenv unsafe.Pointer // pointer to C function +var _cgo_unsetenv unsafe.Pointer // pointer to C function // Update the C environment if cgo is loaded. // Called from syscall.Setenv. func syscall_setenv_c(k string, v string) { - if _cgo_setenv == 0 { + if _cgo_setenv == nil { return } arg := [2]unsafe.Pointer{cstring(k), cstring(v)} @@ -48,7 +46,7 @@ func syscall_setenv_c(k string, v string) { // Update the C environment if cgo is loaded. // Called from syscall.unsetenv. func syscall_unsetenv_c(k string) { - if _cgo_unsetenv == 0 { + if _cgo_unsetenv == nil { return } arg := [1]unsafe.Pointer{cstring(k)} diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index be352557f..e871236e4 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -34,21 +34,11 @@ func lfstackpush_m() func lfstackpop_m() func LFStackPush(head *uint64, node *LFNode) { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(head) - mp.ptrarg[1] = unsafe.Pointer(node) - onM(lfstackpush_m) - releasem(mp) + lfstackpush(head, (*lfnode)(unsafe.Pointer(node))) } func LFStackPop(head *uint64) *LFNode { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(head) - onM(lfstackpop_m) - node := (*LFNode)(unsafe.Pointer(mp.ptrarg[0])) - mp.ptrarg[0] = nil - releasem(mp) - return node + return (*LFNode)(unsafe.Pointer(lfstackpop(head))) } type ParFor struct { @@ -68,69 +58,49 @@ func parfordo_m() func parforiters_m() func NewParFor(nthrmax uint32) *ParFor { - mp := acquirem() - mp.scalararg[0] = uintptr(nthrmax) - onM(newparfor_m) - desc := (*ParFor)(mp.ptrarg[0]) - mp.ptrarg[0] = nil - releasem(mp) + var desc *ParFor + systemstack(func() { + desc = (*ParFor)(unsafe.Pointer(parforalloc(nthrmax))) + }) return desc } func ParForSetup(desc *ParFor, nthr, n uint32, ctx *byte, wait bool, body func(*ParFor, uint32)) { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(desc) - mp.ptrarg[1] = unsafe.Pointer(ctx) - mp.ptrarg[2] = unsafe.Pointer(funcPC(body)) // TODO(rsc): Should be a scalar. - mp.scalararg[0] = uintptr(nthr) - mp.scalararg[1] = uintptr(n) - mp.scalararg[2] = 0 - if wait { - mp.scalararg[2] = 1 - } - onM(parforsetup_m) - releasem(mp) + systemstack(func() { + parforsetup((*parfor)(unsafe.Pointer(desc)), nthr, n, unsafe.Pointer(ctx), wait, + *(*func(*parfor, uint32))(unsafe.Pointer(&body))) + }) } func ParForDo(desc *ParFor) { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(desc) - onM(parfordo_m) - releasem(mp) + systemstack(func() { + parfordo((*parfor)(unsafe.Pointer(desc))) + }) } func ParForIters(desc *ParFor, tid uint32) (uint32, uint32) { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(desc) - mp.scalararg[0] = uintptr(tid) - onM(parforiters_m) - begin := uint32(mp.scalararg[0]) - end := uint32(mp.scalararg[1]) - releasem(mp) - return begin, end + desc1 := (*parfor)(unsafe.Pointer(desc)) + pos := desc_thr_index(desc1, tid).pos + return uint32(pos), uint32(pos >> 32) } -// in mgc0.c -//go:noescape -func getgcmask(data unsafe.Pointer, typ *_type, array **byte, len *uint) - func GCMask(x interface{}) (ret []byte) { e := (*eface)(unsafe.Pointer(&x)) s := (*slice)(unsafe.Pointer(&ret)) - onM(func() { - getgcmask(e.data, e._type, &s.array, &s.len) + systemstack(func() { + var len uintptr + getgcmask(e.data, e._type, &s.array, &len) + s.len = uint(len) s.cap = s.len }) return } -func testSchedLocalQueue() -func testSchedLocalQueueSteal() func RunSchedLocalQueueTest() { - onM(testSchedLocalQueue) + systemstack(testSchedLocalQueue) } func RunSchedLocalQueueStealTest() { - onM(testSchedLocalQueueSteal) + systemstack(testSchedLocalQueueSteal) } var HaveGoodHash = haveGoodHash @@ -149,13 +119,9 @@ func GogoBytes() int32 { return _RuntimeGogoBytes } -// in string.c -//go:noescape -func gostringw(w *uint16) string - // entry point for testing func GostringW(w []uint16) (s string) { - onM(func() { + systemstack(func() { s = gostringw(&w[0]) }) return diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 6cc5df810..34fdeb2b4 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -112,7 +112,8 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) { if xpc > f.entry && (g == nil || g.entry != funcPC(sigpanic)) { xpc-- } - line = int(funcline(f, xpc, &file)) + file, line32 := funcline(f, xpc) + line = int(line32) ok = true return } diff --git a/src/runtime/float.c b/src/runtime/float.c deleted file mode 100644 index 42082e434..000000000 --- a/src/runtime/float.c +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" - -// used as float64 via runtime· names -uint64 ·nan = 0x7FF8000000000001ULL; -uint64 ·posinf = 0x7FF0000000000000ULL; -uint64 ·neginf = 0xFFF0000000000000ULL; diff --git a/src/runtime/funcdata.h b/src/runtime/funcdata.h index d6c14fcb4..ce62dabe3 100644 --- a/src/runtime/funcdata.h +++ b/src/runtime/funcdata.h @@ -3,9 +3,10 @@ // license that can be found in the LICENSE file. // This file defines the IDs for PCDATA and FUNCDATA instructions -// in Go binaries. It is included by both C and assembly, so it must -// be written using #defines. It is included by the runtime package -// as well as the compilers. +// in Go binaries. It is included by assembly sources, so it must +// be written using #defines. +// +// The Go compiler also #includes this file, for now. // // symtab.go also contains a copy of these constants. @@ -50,8 +51,7 @@ /*c2go enum { - PCDATA_ArgSize = 0, - PCDATA_StackMapIndex = 1, + PCDATA_StackMapIndex = 0, FUNCDATA_ArgsPointerMaps = 0, FUNCDATA_LocalsPointerMaps = 1, FUNCDATA_DeadValueMaps = 2, diff --git a/src/runtime/futex_test.go b/src/runtime/futex_test.go index f57fc52b8..b85249a54 100644 --- a/src/runtime/futex_test.go +++ b/src/runtime/futex_test.go @@ -44,9 +44,9 @@ func TestFutexsleep(t *testing.T) { start := time.Now() for _, tt := range futexsleepTests { go func(tt futexsleepTest) { - runtime.Entersyscall() + runtime.Entersyscall(0) runtime.Futexsleep(&tt.mtx, tt.mtx, tt.ns) - runtime.Exitsyscall() + runtime.Exitsyscall(0) tt.ch <- tt }(tt) } diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 88f6703f9..2b45c8184 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -62,12 +62,10 @@ func verifyGCInfo(t *testing.T, name string, p interface{}, mask0 []byte) { func nonStackInfo(mask []byte) []byte { // BitsDead is replaced with BitsScalar everywhere except stacks. mask1 := make([]byte, len(mask)) - mw := false for i, v := range mask { - if !mw && v == BitsDead { + if v == BitsDead { v = BitsScalar } - mw = !mw && v == BitsMultiWord mask1[i] = v } return mask1 @@ -84,7 +82,6 @@ const ( BitsDead = iota BitsScalar BitsPointer - BitsMultiWord ) const ( @@ -137,7 +134,7 @@ func infoBigStruct() []byte { BitsScalar, BitsScalar, BitsScalar, BitsScalar, // t int; y uint16; u uint64 BitsPointer, BitsDead, // i string } - case "amd64": + case "amd64", "power64", "power64le": return []byte{ BitsPointer, // q *int BitsScalar, BitsScalar, BitsScalar, // w byte; e [17]byte @@ -188,6 +185,6 @@ var ( infoString = []byte{BitsPointer, BitsDead} infoSlice = []byte{BitsPointer, BitsDead, BitsDead} - infoEface = []byte{BitsMultiWord, BitsEface} - infoIface = []byte{BitsMultiWord, BitsIface} + infoEface = []byte{BitsPointer, BitsPointer} + infoIface = []byte{BitsPointer, BitsPointer} ) diff --git a/src/runtime/gengoos.go b/src/runtime/gengoos.go new file mode 100644 index 000000000..029575bee --- /dev/null +++ b/src/runtime/gengoos.go @@ -0,0 +1,82 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "strconv" + "strings" +) + +var gooses, goarches []string + +func main() { + data, err := ioutil.ReadFile("../go/build/syslist.go") + if err != nil { + log.Fatal(err) + } + const ( + goosPrefix = `const goosList = ` + goarchPrefix = `const goarchList = ` + ) + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, goosPrefix) { + text, err := strconv.Unquote(strings.TrimPrefix(line, goosPrefix)) + if err != nil { + log.Fatalf("parsing goosList %#q: %v", strings.TrimPrefix(line, goosPrefix), err) + } + gooses = strings.Fields(text) + } + if strings.HasPrefix(line, goarchPrefix) { + text, err := strconv.Unquote(strings.TrimPrefix(line, goarchPrefix)) + if err != nil { + log.Fatal("parsing goarchList: %v", err) + } + goarches = strings.Fields(text) + } + } + + for _, target := range gooses { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// generated by gengoos.go using 'go generate'\n\n") + fmt.Fprintf(&buf, "// +build %s\n\n", target) // usually redundant, but not always; see linux vs android + fmt.Fprintf(&buf, "package runtime\n\n") + fmt.Fprintf(&buf, "const theGoos = `%s`\n\n", target) + for _, goos := range gooses { + value := 0 + if goos == target { + value = 1 + } + fmt.Fprintf(&buf, "const goos_%s = %d\n", goos, value) + } + err := ioutil.WriteFile("zgoos_"+target+".go", buf.Bytes(), 0666) + if err != nil { + log.Fatal(err) + } + } + + for _, target := range goarches { + var buf bytes.Buffer + fmt.Fprintf(&buf, "// generated by gengoos.go using 'go generate'\n\n") + fmt.Fprintf(&buf, "package runtime\n\n") + fmt.Fprintf(&buf, "const theGoarch = `%s`\n\n", target) + for _, goarch := range goarches { + value := 0 + if goarch == target { + value = 1 + } + fmt.Fprintf(&buf, "const goarch_%s = %d\n", goarch, value) + } + err := ioutil.WriteFile("zgoarch_"+target+".go", buf.Bytes(), 0666) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/src/runtime/go_tls.h b/src/runtime/go_tls.h new file mode 100644 index 000000000..6a707cf1e --- /dev/null +++ b/src/runtime/go_tls.h @@ -0,0 +1,22 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifdef GOARCH_arm +#define LR R14 +#endif + +#ifdef GOARCH_amd64 +#define get_tls(r) MOVQ TLS, r +#define g(r) 0(r)(TLS*1) +#endif + +#ifdef GOARCH_amd64p32 +#define get_tls(r) MOVL TLS, r +#define g(r) 0(r)(TLS*1) +#endif + +#ifdef GOARCH_386 +#define get_tls(r) MOVL TLS, r +#define g(r) 0(r)(TLS*1) +#endif diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c deleted file mode 100644 index 7eba8c005..000000000 --- a/src/runtime/heapdump.c +++ /dev/null @@ -1,864 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Implementation of runtime/debug.WriteHeapDump. Writes all -// objects in the heap plus additional info (roots, threads, -// finalizers, etc.) to a file. - -// The format of the dumped file is described at -// http://golang.org/s/go14heapdump. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "mgc0.h" -#include "type.h" -#include "typekind.h" -#include "funcdata.h" -#include "zaexperiment.h" -#include "textflag.h" - -extern byte runtime·data[]; -extern byte runtime·edata[]; -extern byte runtime·bss[]; -extern byte runtime·ebss[]; - -enum { - FieldKindEol = 0, - FieldKindPtr = 1, - FieldKindIface = 2, - FieldKindEface = 3, - - TagEOF = 0, - TagObject = 1, - TagOtherRoot = 2, - TagType = 3, - TagGoRoutine = 4, - TagStackFrame = 5, - TagParams = 6, - TagFinalizer = 7, - TagItab = 8, - TagOSThread = 9, - TagMemStats = 10, - TagQueuedFinalizer = 11, - TagData = 12, - TagBss = 13, - TagDefer = 14, - TagPanic = 15, - TagMemProf = 16, - TagAllocSample = 17, -}; - -static uintptr* playgcprog(uintptr offset, uintptr *prog, void (*callback)(void*,uintptr,uintptr), void *arg); -static void dumpfields(BitVector bv); -static void dumpbvtypes(BitVector *bv, byte *base); -static BitVector makeheapobjbv(byte *p, uintptr size); - -// fd to write the dump to. -static uintptr dumpfd; - -#pragma dataflag NOPTR /* tmpbuf not a heap pointer at least */ -static byte *tmpbuf; -static uintptr tmpbufsize; - -// buffer of pending write data -enum { - BufSize = 4096, -}; -#pragma dataflag NOPTR -static byte buf[BufSize]; -static uintptr nbuf; - -static void -write(byte *data, uintptr len) -{ - if(len + nbuf <= BufSize) { - runtime·memmove(buf + nbuf, data, len); - nbuf += len; - return; - } - runtime·write(dumpfd, buf, nbuf); - if(len >= BufSize) { - runtime·write(dumpfd, data, len); - nbuf = 0; - } else { - runtime·memmove(buf, data, len); - nbuf = len; - } -} - -static void -flush(void) -{ - runtime·write(dumpfd, buf, nbuf); - nbuf = 0; -} - -// Cache of types that have been serialized already. -// We use a type's hash field to pick a bucket. -// Inside a bucket, we keep a list of types that -// have been serialized so far, most recently used first. -// Note: when a bucket overflows we may end up -// serializing a type more than once. That's ok. -enum { - TypeCacheBuckets = 256, // must be a power of 2 - TypeCacheAssoc = 4, -}; -typedef struct TypeCacheBucket TypeCacheBucket; -struct TypeCacheBucket { - Type *t[TypeCacheAssoc]; -}; -#pragma dataflag NOPTR /* only initialized and used while world is stopped */ -static TypeCacheBucket typecache[TypeCacheBuckets]; - -// dump a uint64 in a varint format parseable by encoding/binary -static void -dumpint(uint64 v) -{ - byte buf[10]; - int32 n; - n = 0; - while(v >= 0x80) { - buf[n++] = v | 0x80; - v >>= 7; - } - buf[n++] = v; - write(buf, n); -} - -static void -dumpbool(bool b) -{ - dumpint(b ? 1 : 0); -} - -// dump varint uint64 length followed by memory contents -static void -dumpmemrange(byte *data, uintptr len) -{ - dumpint(len); - write(data, len); -} - -static void -dumpstr(String s) -{ - dumpmemrange(s.str, s.len); -} - -static void -dumpcstr(int8 *c) -{ - dumpmemrange((byte*)c, runtime·findnull((byte*)c)); -} - -// dump information for a type -static void -dumptype(Type *t) -{ - TypeCacheBucket *b; - int32 i, j; - - if(t == nil) { - return; - } - - // If we've definitely serialized the type before, - // no need to do it again. - b = &typecache[t->hash & (TypeCacheBuckets-1)]; - if(t == b->t[0]) return; - for(i = 1; i < TypeCacheAssoc; i++) { - if(t == b->t[i]) { - // Move-to-front - for(j = i; j > 0; j--) { - b->t[j] = b->t[j-1]; - } - b->t[0] = t; - return; - } - } - // Might not have been dumped yet. Dump it and - // remember we did so. - for(j = TypeCacheAssoc-1; j > 0; j--) { - b->t[j] = b->t[j-1]; - } - b->t[0] = t; - - // dump the type - dumpint(TagType); - dumpint((uintptr)t); - dumpint(t->size); - if(t->x == nil || t->x->pkgPath == nil || t->x->name == nil) { - dumpstr(*t->string); - } else { - dumpint(t->x->pkgPath->len + 1 + t->x->name->len); - write(t->x->pkgPath->str, t->x->pkgPath->len); - write((byte*)".", 1); - write(t->x->name->str, t->x->name->len); - } - dumpbool((t->kind & KindDirectIface) == 0 || (t->kind & KindNoPointers) == 0); -} - -// dump an object -static void -dumpobj(byte *obj, uintptr size, BitVector bv) -{ - dumpbvtypes(&bv, obj); - dumpint(TagObject); - dumpint((uintptr)obj); - dumpmemrange(obj, size); - dumpfields(bv); -} - -static void -dumpotherroot(int8 *description, byte *to) -{ - dumpint(TagOtherRoot); - dumpcstr(description); - dumpint((uintptr)to); -} - -static void -dumpfinalizer(byte *obj, FuncVal *fn, Type* fint, PtrType *ot) -{ - dumpint(TagFinalizer); - dumpint((uintptr)obj); - dumpint((uintptr)fn); - dumpint((uintptr)fn->fn); - dumpint((uintptr)fint); - dumpint((uintptr)ot); -} - -typedef struct ChildInfo ChildInfo; -struct ChildInfo { - // Information passed up from the callee frame about - // the layout of the outargs region. - uintptr argoff; // where the arguments start in the frame - uintptr arglen; // size of args region - BitVector args; // if args.n >= 0, pointer map of args region - - byte *sp; // callee sp - uintptr depth; // depth in call stack (0 == most recent) -}; - -// dump kinds & offsets of interesting fields in bv -static void -dumpbv(BitVector *bv, uintptr offset) -{ - uintptr i; - - for(i = 0; i < bv->n; i += BitsPerPointer) { - switch(bv->bytedata[i/8] >> i%8 & 3) { - case BitsDead: - // BitsDead has already been processed in makeheapobjbv. - // We should only see it in stack maps, in which case we should continue processing. - break; - case BitsScalar: - break; - case BitsPointer: - dumpint(FieldKindPtr); - dumpint(offset + i / BitsPerPointer * PtrSize); - break; - case BitsMultiWord: - switch(bv->bytedata[(i+BitsPerPointer)/8] >> (i+BitsPerPointer)%8 & 3) { - default: - runtime·throw("unexpected garbage collection bits"); - case BitsIface: - dumpint(FieldKindIface); - dumpint(offset + i / BitsPerPointer * PtrSize); - i += BitsPerPointer; - break; - case BitsEface: - dumpint(FieldKindEface); - dumpint(offset + i / BitsPerPointer * PtrSize); - i += BitsPerPointer; - break; - } - } - } -} - -static bool -dumpframe(Stkframe *s, void *arg) -{ - Func *f; - ChildInfo *child; - uintptr pc, off, size; - int32 pcdata; - StackMap *stackmap; - int8 *name; - BitVector bv; - - child = (ChildInfo*)arg; - f = s->fn; - - // Figure out what we can about our stack map - pc = s->pc; - if(pc != f->entry) - pc--; - pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, pc); - if(pcdata == -1) { - // We do not have a valid pcdata value but there might be a - // stackmap for this function. It is likely that we are looking - // at the function prologue, assume so and hope for the best. - pcdata = 0; - } - stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); - - // Dump any types we will need to resolve Efaces. - if(child->args.n >= 0) - dumpbvtypes(&child->args, (byte*)s->sp + child->argoff); - if(stackmap != nil && stackmap->n > 0) { - bv = runtime·stackmapdata(stackmap, pcdata); - dumpbvtypes(&bv, (byte*)(s->varp - bv.n / BitsPerPointer * PtrSize)); - } else { - bv.n = -1; - } - - // Dump main body of stack frame. - dumpint(TagStackFrame); - dumpint(s->sp); // lowest address in frame - dumpint(child->depth); // # of frames deep on the stack - dumpint((uintptr)child->sp); // sp of child, or 0 if bottom of stack - dumpmemrange((byte*)s->sp, s->fp - s->sp); // frame contents - dumpint(f->entry); - dumpint(s->pc); - dumpint(s->continpc); - name = runtime·funcname(f); - if(name == nil) - name = "unknown function"; - dumpcstr(name); - - // Dump fields in the outargs section - if(child->args.n >= 0) { - dumpbv(&child->args, child->argoff); - } else { - // conservative - everything might be a pointer - for(off = child->argoff; off < child->argoff + child->arglen; off += PtrSize) { - dumpint(FieldKindPtr); - dumpint(off); - } - } - - // Dump fields in the local vars section - if(stackmap == nil) { - // No locals information, dump everything. - for(off = child->arglen; off < s->varp - s->sp; off += PtrSize) { - dumpint(FieldKindPtr); - dumpint(off); - } - } else if(stackmap->n < 0) { - // Locals size information, dump just the locals. - size = -stackmap->n; - for(off = s->varp - size - s->sp; off < s->varp - s->sp; off += PtrSize) { - dumpint(FieldKindPtr); - dumpint(off); - } - } else if(stackmap->n > 0) { - // Locals bitmap information, scan just the pointers in - // locals. - dumpbv(&bv, s->varp - bv.n / BitsPerPointer * PtrSize - s->sp); - } - dumpint(FieldKindEol); - - // Record arg info for parent. - child->argoff = s->argp - s->fp; - child->arglen = s->arglen; - child->sp = (byte*)s->sp; - child->depth++; - stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); - if(stackmap != nil) - child->args = runtime·stackmapdata(stackmap, pcdata); - else - child->args.n = -1; - return true; -} - -static void -dumpgoroutine(G *gp) -{ - uintptr sp, pc, lr; - ChildInfo child; - Defer *d; - Panic *p; - bool (*fn)(Stkframe*, void*); - - if(gp->syscallsp != (uintptr)nil) { - sp = gp->syscallsp; - pc = gp->syscallpc; - lr = 0; - } else { - sp = gp->sched.sp; - pc = gp->sched.pc; - lr = gp->sched.lr; - } - - dumpint(TagGoRoutine); - dumpint((uintptr)gp); - dumpint((uintptr)sp); - dumpint(gp->goid); - dumpint(gp->gopc); - dumpint(runtime·readgstatus(gp)); - dumpbool(gp->issystem); - dumpbool(false); // isbackground - dumpint(gp->waitsince); - dumpstr(gp->waitreason); - dumpint((uintptr)gp->sched.ctxt); - dumpint((uintptr)gp->m); - dumpint((uintptr)gp->defer); - dumpint((uintptr)gp->panic); - - // dump stack - child.args.n = -1; - child.arglen = 0; - child.sp = nil; - child.depth = 0; - fn = dumpframe; - runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, &fn, &child, 0); - - // dump defer & panic records - for(d = gp->defer; d != nil; d = d->link) { - dumpint(TagDefer); - dumpint((uintptr)d); - dumpint((uintptr)gp); - dumpint((uintptr)d->argp); - dumpint((uintptr)d->pc); - dumpint((uintptr)d->fn); - dumpint((uintptr)d->fn->fn); - dumpint((uintptr)d->link); - } - for (p = gp->panic; p != nil; p = p->link) { - dumpint(TagPanic); - dumpint((uintptr)p); - dumpint((uintptr)gp); - dumpint((uintptr)p->arg.type); - dumpint((uintptr)p->arg.data); - dumpint(0); // was p->defer, no longer recorded - dumpint((uintptr)p->link); - } -} - -static void -dumpgs(void) -{ - G *gp; - uint32 i; - uint32 status; - - // goroutines & stacks - for(i = 0; i < runtime·allglen; i++) { - gp = runtime·allg[i]; - status = runtime·readgstatus(gp); // The world is stopped so gp will not be in a scan state. - switch(status){ - default: - runtime·printf("runtime: unexpected G.status %d\n", status); - runtime·throw("dumpgs in STW - bad status"); - case Gdead: - break; - case Grunnable: - case Gsyscall: - case Gwaiting: - dumpgoroutine(gp); - break; - } - } -} - -static void -finq_callback(FuncVal *fn, byte *obj, uintptr nret, Type *fint, PtrType *ot) -{ - dumpint(TagQueuedFinalizer); - dumpint((uintptr)obj); - dumpint((uintptr)fn); - dumpint((uintptr)fn->fn); - dumpint((uintptr)fint); - dumpint((uintptr)ot); - USED(&nret); -} - - -static void -dumproots(void) -{ - MSpan *s, **allspans; - uint32 spanidx; - Special *sp; - SpecialFinalizer *spf; - byte *p; - - // data segment - dumpbvtypes(&runtime·gcdatamask, runtime·data); - dumpint(TagData); - dumpint((uintptr)runtime·data); - dumpmemrange(runtime·data, runtime·edata - runtime·data); - dumpfields(runtime·gcdatamask); - - // bss segment - dumpbvtypes(&runtime·gcbssmask, runtime·bss); - dumpint(TagBss); - dumpint((uintptr)runtime·bss); - dumpmemrange(runtime·bss, runtime·ebss - runtime·bss); - dumpfields(runtime·gcbssmask); - - // MSpan.types - allspans = runtime·mheap.allspans; - for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { - s = allspans[spanidx]; - if(s->state == MSpanInUse) { - // Finalizers - for(sp = s->specials; sp != nil; sp = sp->next) { - if(sp->kind != KindSpecialFinalizer) - continue; - spf = (SpecialFinalizer*)sp; - p = (byte*)((s->start << PageShift) + spf->special.offset); - dumpfinalizer(p, spf->fn, spf->fint, spf->ot); - } - } - } - - // Finalizer queue - runtime·iterate_finq(finq_callback); -} - -// Bit vector of free marks. -// Needs to be as big as the largest number of objects per span. -#pragma dataflag NOPTR -static byte free[PageSize/8]; - -static void -dumpobjs(void) -{ - uintptr i, j, size, n; - MSpan *s; - MLink *l; - byte *p; - - for(i = 0; i < runtime·mheap.nspan; i++) { - s = runtime·mheap.allspans[i]; - if(s->state != MSpanInUse) - continue; - p = (byte*)(s->start << PageShift); - size = s->elemsize; - n = (s->npages << PageShift) / size; - if(n > nelem(free)) - runtime·throw("free array doesn't have enough entries"); - for(l = s->freelist; l != nil; l = l->next) - free[((byte*)l - p) / size] = true; - for(j = 0; j < n; j++, p += size) { - if(free[j]) { - free[j] = false; - continue; - } - dumpobj(p, size, makeheapobjbv(p, size)); - } - } -} - -static void -dumpparams(void) -{ - byte *x; - - dumpint(TagParams); - x = (byte*)1; - if(*(byte*)&x == 1) - dumpbool(false); // little-endian ptrs - else - dumpbool(true); // big-endian ptrs - dumpint(PtrSize); - dumpint((uintptr)runtime·mheap.arena_start); - dumpint((uintptr)runtime·mheap.arena_used); - dumpint(thechar); - dumpcstr(GOEXPERIMENT); - dumpint(runtime·ncpu); -} - -static void -itab_callback(Itab *tab) -{ - Type *t; - - t = tab->type; - // Dump a map from itab* to the type of its data field. - // We want this map so we can deduce types of interface referents. - if((t->kind & KindDirectIface) == 0) { - // indirect - data slot is a pointer to t. - dumptype(t->ptrto); - dumpint(TagItab); - dumpint((uintptr)tab); - dumpint((uintptr)t->ptrto); - } else if((t->kind & KindNoPointers) == 0) { - // t is pointer-like - data slot is a t. - dumptype(t); - dumpint(TagItab); - dumpint((uintptr)tab); - dumpint((uintptr)t); - } else { - // Data slot is a scalar. Dump type just for fun. - // With pointer-only interfaces, this shouldn't happen. - dumptype(t); - dumpint(TagItab); - dumpint((uintptr)tab); - dumpint((uintptr)t); - } -} - -static void -dumpitabs(void) -{ - void (*fn)(Itab*); - - fn = itab_callback; - runtime·iterate_itabs(&fn); -} - -static void -dumpms(void) -{ - M *mp; - - for(mp = runtime·allm; mp != nil; mp = mp->alllink) { - dumpint(TagOSThread); - dumpint((uintptr)mp); - dumpint(mp->id); - dumpint(mp->procid); - } -} - -static void -dumpmemstats(void) -{ - int32 i; - - dumpint(TagMemStats); - dumpint(mstats.alloc); - dumpint(mstats.total_alloc); - dumpint(mstats.sys); - dumpint(mstats.nlookup); - dumpint(mstats.nmalloc); - dumpint(mstats.nfree); - dumpint(mstats.heap_alloc); - dumpint(mstats.heap_sys); - dumpint(mstats.heap_idle); - dumpint(mstats.heap_inuse); - dumpint(mstats.heap_released); - dumpint(mstats.heap_objects); - dumpint(mstats.stacks_inuse); - dumpint(mstats.stacks_sys); - dumpint(mstats.mspan_inuse); - dumpint(mstats.mspan_sys); - dumpint(mstats.mcache_inuse); - dumpint(mstats.mcache_sys); - dumpint(mstats.buckhash_sys); - dumpint(mstats.gc_sys); - dumpint(mstats.other_sys); - dumpint(mstats.next_gc); - dumpint(mstats.last_gc); - dumpint(mstats.pause_total_ns); - for(i = 0; i < 256; i++) - dumpint(mstats.pause_ns[i]); - dumpint(mstats.numgc); -} - -static void -dumpmemprof_callback(Bucket *b, uintptr nstk, uintptr *stk, uintptr size, uintptr allocs, uintptr frees) -{ - uintptr i, pc; - Func *f; - byte buf[20]; - String file; - int32 line; - - dumpint(TagMemProf); - dumpint((uintptr)b); - dumpint(size); - dumpint(nstk); - for(i = 0; i < nstk; i++) { - pc = stk[i]; - f = runtime·findfunc(pc); - if(f == nil) { - runtime·snprintf(buf, sizeof(buf), "%X", (uint64)pc); - dumpcstr((int8*)buf); - dumpcstr("?"); - dumpint(0); - } else { - dumpcstr(runtime·funcname(f)); - // TODO: Why do we need to back up to a call instruction here? - // Maybe profiler should do this. - if(i > 0 && pc > f->entry) { - if(thechar == '6' || thechar == '8') - pc--; - else - pc -= 4; // arm, etc - } - line = runtime·funcline(f, pc, &file); - dumpstr(file); - dumpint(line); - } - } - dumpint(allocs); - dumpint(frees); -} - -static void -dumpmemprof(void) -{ - MSpan *s, **allspans; - uint32 spanidx; - Special *sp; - SpecialProfile *spp; - byte *p; - void (*fn)(Bucket*, uintptr, uintptr*, uintptr, uintptr, uintptr); - - fn = dumpmemprof_callback; - runtime·iterate_memprof(&fn); - - allspans = runtime·mheap.allspans; - for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { - s = allspans[spanidx]; - if(s->state != MSpanInUse) - continue; - for(sp = s->specials; sp != nil; sp = sp->next) { - if(sp->kind != KindSpecialProfile) - continue; - spp = (SpecialProfile*)sp; - p = (byte*)((s->start << PageShift) + spp->special.offset); - dumpint(TagAllocSample); - dumpint((uintptr)p); - dumpint((uintptr)spp->b); - } - } -} - -static void -mdump(void) -{ - byte *hdr; - uintptr i; - MSpan *s; - - // make sure we're done sweeping - for(i = 0; i < runtime·mheap.nspan; i++) { - s = runtime·mheap.allspans[i]; - if(s->state == MSpanInUse) - runtime·MSpan_EnsureSwept(s); - } - - runtime·memclr((byte*)&typecache[0], sizeof(typecache)); - hdr = (byte*)"go1.4 heap dump\n"; - write(hdr, runtime·findnull(hdr)); - dumpparams(); - dumpitabs(); - dumpobjs(); - dumpgs(); - dumpms(); - dumproots(); - dumpmemstats(); - dumpmemprof(); - dumpint(TagEOF); - flush(); -} - -void -runtime·writeheapdump_m(void) -{ - uintptr fd; - - fd = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - - runtime·casgstatus(g->m->curg, Grunning, Gwaiting); - g->waitreason = runtime·gostringnocopy((byte*)"dumping heap"); - - // Update stats so we can dump them. - // As a side effect, flushes all the MCaches so the MSpan.freelist - // lists contain all the free objects. - runtime·updatememstats(nil); - - // Set dump file. - dumpfd = fd; - - // Call dump routine. - mdump(); - - // Reset dump file. - dumpfd = 0; - if(tmpbuf != nil) { - runtime·SysFree(tmpbuf, tmpbufsize, &mstats.other_sys); - tmpbuf = nil; - tmpbufsize = 0; - } - - runtime·casgstatus(g->m->curg, Gwaiting, Grunning); -} - -// dumpint() the kind & offset of each field in an object. -static void -dumpfields(BitVector bv) -{ - dumpbv(&bv, 0); - dumpint(FieldKindEol); -} - -// The heap dump reader needs to be able to disambiguate -// Eface entries. So it needs to know every type that might -// appear in such an entry. The following routine accomplishes that. - -// Dump all the types that appear in the type field of -// any Eface described by this bit vector. -static void -dumpbvtypes(BitVector *bv, byte *base) -{ - uintptr i; - - for(i = 0; i < bv->n; i += BitsPerPointer) { - if((bv->bytedata[i/8] >> i%8 & 3) != BitsMultiWord) - continue; - switch(bv->bytedata[(i+BitsPerPointer)/8] >> (i+BitsPerPointer)%8 & 3) { - default: - runtime·throw("unexpected garbage collection bits"); - case BitsIface: - i += BitsPerPointer; - break; - case BitsEface: - dumptype(*(Type**)(base + i / BitsPerPointer * PtrSize)); - i += BitsPerPointer; - break; - } - } -} - -static BitVector -makeheapobjbv(byte *p, uintptr size) -{ - uintptr off, nptr, i; - byte shift, *bitp, bits; - bool mw; - - // Extend the temp buffer if necessary. - nptr = size/PtrSize; - if(tmpbufsize < nptr*BitsPerPointer/8+1) { - if(tmpbuf != nil) - runtime·SysFree(tmpbuf, tmpbufsize, &mstats.other_sys); - tmpbufsize = nptr*BitsPerPointer/8+1; - tmpbuf = runtime·sysAlloc(tmpbufsize, &mstats.other_sys); - if(tmpbuf == nil) - runtime·throw("heapdump: out of memory"); - } - - // Copy and compact the bitmap. - mw = false; - for(i = 0; i < nptr; i++) { - off = (uintptr*)(p + i*PtrSize) - (uintptr*)runtime·mheap.arena_start; - bitp = runtime·mheap.arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - bits = (*bitp >> (shift + 2)) & BitsMask; - if(!mw && bits == BitsDead) - break; // end of heap object - mw = !mw && bits == BitsMultiWord; - tmpbuf[i*BitsPerPointer/8] &= ~(BitsMask<<((i*BitsPerPointer)%8)); - tmpbuf[i*BitsPerPointer/8] |= bits<<((i*BitsPerPointer)%8); - } - return (BitVector){i*BitsPerPointer, tmpbuf}; -} diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go new file mode 100644 index 000000000..0c1a60c8b --- /dev/null +++ b/src/runtime/heapdump.go @@ -0,0 +1,729 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implementation of runtime/debug.WriteHeapDump. Writes all +// objects in the heap plus additional info (roots, threads, +// finalizers, etc.) to a file. + +// The format of the dumped file is described at +// http://golang.org/s/go14heapdump. + +package runtime + +import "unsafe" + +const ( + fieldKindEol = 0 + fieldKindPtr = 1 + fieldKindIface = 2 + fieldKindEface = 3 + tagEOF = 0 + tagObject = 1 + tagOtherRoot = 2 + tagType = 3 + tagGoroutine = 4 + tagStackFrame = 5 + tagParams = 6 + tagFinalizer = 7 + tagItab = 8 + tagOSThread = 9 + tagMemStats = 10 + tagQueuedFinalizer = 11 + tagData = 12 + tagBSS = 13 + tagDefer = 14 + tagPanic = 15 + tagMemProf = 16 + tagAllocSample = 17 +) + +var dumpfd uintptr // fd to write the dump to. +var tmpbuf []byte + +// buffer of pending write data +const ( + bufSize = 4096 +) + +var buf [bufSize]byte +var nbuf uintptr + +func dwrite(data unsafe.Pointer, len uintptr) { + if len == 0 { + return + } + if nbuf+len <= bufSize { + copy(buf[nbuf:], (*[bufSize]byte)(data)[:len]) + nbuf += len + return + } + + write(dumpfd, (unsafe.Pointer)(&buf), int32(nbuf)) + if len >= bufSize { + write(dumpfd, data, int32(len)) + nbuf = 0 + } else { + copy(buf[:], (*[bufSize]byte)(data)[:len]) + nbuf = len + } +} + +func dwritebyte(b byte) { + dwrite(unsafe.Pointer(&b), 1) +} + +func flush() { + write(dumpfd, (unsafe.Pointer)(&buf), int32(nbuf)) + nbuf = 0 +} + +// Cache of types that have been serialized already. +// We use a type's hash field to pick a bucket. +// Inside a bucket, we keep a list of types that +// have been serialized so far, most recently used first. +// Note: when a bucket overflows we may end up +// serializing a type more than once. That's ok. +const ( + typeCacheBuckets = 256 + typeCacheAssoc = 4 +) + +type typeCacheBucket struct { + t [typeCacheAssoc]*_type +} + +var typecache [typeCacheBuckets]typeCacheBucket + +// dump a uint64 in a varint format parseable by encoding/binary +func dumpint(v uint64) { + var buf [10]byte + var n int + for v >= 0x80 { + buf[n] = byte(v | 0x80) + n++ + v >>= 7 + } + buf[n] = byte(v) + n++ + dwrite(unsafe.Pointer(&buf), uintptr(n)) +} + +func dumpbool(b bool) { + if b { + dumpint(1) + } else { + dumpint(0) + } +} + +// dump varint uint64 length followed by memory contents +func dumpmemrange(data unsafe.Pointer, len uintptr) { + dumpint(uint64(len)) + dwrite(data, len) +} + +func dumpslice(b []byte) { + dumpint(uint64(len(b))) + if len(b) > 0 { + dwrite(unsafe.Pointer(&b[0]), uintptr(len(b))) + } +} + +func dumpstr(s string) { + sp := (*stringStruct)(unsafe.Pointer(&s)) + dumpmemrange(sp.str, uintptr(sp.len)) +} + +// dump information for a type +func dumptype(t *_type) { + if t == nil { + return + } + + // If we've definitely serialized the type before, + // no need to do it again. + b := &typecache[t.hash&(typeCacheBuckets-1)] + if t == b.t[0] { + return + } + for i := 1; i < typeCacheAssoc; i++ { + if t == b.t[i] { + // Move-to-front + for j := i; j > 0; j-- { + b.t[j] = b.t[j-1] + } + b.t[0] = t + return + } + } + + // Might not have been dumped yet. Dump it and + // remember we did so. + for j := typeCacheAssoc - 1; j > 0; j-- { + b.t[j] = b.t[j-1] + } + b.t[0] = t + + // dump the type + dumpint(tagType) + dumpint(uint64(uintptr(unsafe.Pointer(t)))) + dumpint(uint64(t.size)) + if t.x == nil || t.x.pkgpath == nil || t.x.name == nil { + dumpstr(*t._string) + } else { + pkgpath := (*stringStruct)(unsafe.Pointer(&t.x.pkgpath)) + name := (*stringStruct)(unsafe.Pointer(&t.x.name)) + dumpint(uint64(uintptr(pkgpath.len) + 1 + uintptr(name.len))) + dwrite(pkgpath.str, uintptr(pkgpath.len)) + dwritebyte('.') + dwrite(name.str, uintptr(name.len)) + } + dumpbool(t.kind&kindDirectIface == 0 || t.kind&kindNoPointers == 0) +} + +// dump an object +func dumpobj(obj unsafe.Pointer, size uintptr, bv bitvector) { + dumpbvtypes(&bv, obj) + dumpint(tagObject) + dumpint(uint64(uintptr(obj))) + dumpmemrange(obj, size) + dumpfields(bv) +} + +func dumpotherroot(description string, to unsafe.Pointer) { + dumpint(tagOtherRoot) + dumpstr(description) + dumpint(uint64(uintptr(to))) +} + +func dumpfinalizer(obj unsafe.Pointer, fn *funcval, fint *_type, ot *ptrtype) { + dumpint(tagFinalizer) + dumpint(uint64(uintptr(obj))) + dumpint(uint64(uintptr(unsafe.Pointer(fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(fn.fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(fint)))) + dumpint(uint64(uintptr(unsafe.Pointer(ot)))) +} + +type childInfo struct { + // Information passed up from the callee frame about + // the layout of the outargs region. + argoff uintptr // where the arguments start in the frame + arglen uintptr // size of args region + args bitvector // if args.n >= 0, pointer map of args region + sp *uint8 // callee sp + depth uintptr // depth in call stack (0 == most recent) +} + +// dump kinds & offsets of interesting fields in bv +func dumpbv(cbv *bitvector, offset uintptr) { + bv := gobv(*cbv) + for i := uintptr(0); i < uintptr(bv.n); i += bitsPerPointer { + switch bv.bytedata[i/8] >> (i % 8) & 3 { + default: + gothrow("unexpected pointer bits") + case _BitsDead: + // BitsDead has already been processed in makeheapobjbv. + // We should only see it in stack maps, in which case we should continue processing. + case _BitsScalar: + // ok + case _BitsPointer: + dumpint(fieldKindPtr) + dumpint(uint64(offset + i/_BitsPerPointer*ptrSize)) + } + } +} + +func dumpframe(s *stkframe, arg unsafe.Pointer) bool { + child := (*childInfo)(arg) + f := s.fn + + // Figure out what we can about our stack map + pc := s.pc + if pc != f.entry { + pc-- + } + pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, pc) + if pcdata == -1 { + // We do not have a valid pcdata value but there might be a + // stackmap for this function. It is likely that we are looking + // at the function prologue, assume so and hope for the best. + pcdata = 0 + } + stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) + + // Dump any types we will need to resolve Efaces. + if child.args.n >= 0 { + dumpbvtypes(&child.args, unsafe.Pointer(s.sp+child.argoff)) + } + var bv bitvector + if stkmap != nil && stkmap.n > 0 { + bv = stackmapdata(stkmap, pcdata) + dumpbvtypes(&bv, unsafe.Pointer(s.varp-uintptr(bv.n/_BitsPerPointer*ptrSize))) + } else { + bv.n = -1 + } + + // Dump main body of stack frame. + dumpint(tagStackFrame) + dumpint(uint64(s.sp)) // lowest address in frame + dumpint(uint64(child.depth)) // # of frames deep on the stack + dumpint(uint64(uintptr(unsafe.Pointer(child.sp)))) // sp of child, or 0 if bottom of stack + dumpmemrange(unsafe.Pointer(s.sp), s.fp-s.sp) // frame contents + dumpint(uint64(f.entry)) + dumpint(uint64(s.pc)) + dumpint(uint64(s.continpc)) + name := gofuncname(f) + if name == "" { + name = "unknown function" + } + dumpstr(name) + + // Dump fields in the outargs section + if child.args.n >= 0 { + dumpbv(&child.args, child.argoff) + } else { + // conservative - everything might be a pointer + for off := child.argoff; off < child.argoff+child.arglen; off += ptrSize { + dumpint(fieldKindPtr) + dumpint(uint64(off)) + } + } + + // Dump fields in the local vars section + if stkmap == nil { + // No locals information, dump everything. + for off := child.arglen; off < s.varp-s.sp; off += ptrSize { + dumpint(fieldKindPtr) + dumpint(uint64(off)) + } + } else if stkmap.n < 0 { + // Locals size information, dump just the locals. + size := uintptr(-stkmap.n) + for off := s.varp - size - s.sp; off < s.varp-s.sp; off += ptrSize { + dumpint(fieldKindPtr) + dumpint(uint64(off)) + } + } else if stkmap.n > 0 { + // Locals bitmap information, scan just the pointers in + // locals. + dumpbv(&bv, s.varp-uintptr(bv.n)/_BitsPerPointer*ptrSize-s.sp) + } + dumpint(fieldKindEol) + + // Record arg info for parent. + child.argoff = s.argp - s.fp + child.arglen = s.arglen + child.sp = (*uint8)(unsafe.Pointer(s.sp)) + child.depth++ + stkmap = (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps)) + if stkmap != nil { + child.args = stackmapdata(stkmap, pcdata) + } else { + child.args.n = -1 + } + return true +} + +func dumpgoroutine(gp *g) { + var sp, pc, lr uintptr + if gp.syscallsp != 0 { + sp = gp.syscallsp + pc = gp.syscallpc + lr = 0 + } else { + sp = gp.sched.sp + pc = gp.sched.pc + lr = gp.sched.lr + } + + dumpint(tagGoroutine) + dumpint(uint64(uintptr(unsafe.Pointer(gp)))) + dumpint(uint64(sp)) + dumpint(uint64(gp.goid)) + dumpint(uint64(gp.gopc)) + dumpint(uint64(readgstatus(gp))) + dumpbool(gp.issystem) + dumpbool(false) // isbackground + dumpint(uint64(gp.waitsince)) + dumpstr(gp.waitreason) + dumpint(uint64(uintptr(gp.sched.ctxt))) + dumpint(uint64(uintptr(unsafe.Pointer(gp.m)))) + dumpint(uint64(uintptr(unsafe.Pointer(gp._defer)))) + dumpint(uint64(uintptr(unsafe.Pointer(gp._panic)))) + + // dump stack + var child childInfo + child.args.n = -1 + child.arglen = 0 + child.sp = nil + child.depth = 0 + gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, dumpframe, noescape(unsafe.Pointer(&child)), 0) + + // dump defer & panic records + for d := gp._defer; d != nil; d = d.link { + dumpint(tagDefer) + dumpint(uint64(uintptr(unsafe.Pointer(d)))) + dumpint(uint64(uintptr(unsafe.Pointer(gp)))) + dumpint(uint64(d.argp)) + dumpint(uint64(d.pc)) + dumpint(uint64(uintptr(unsafe.Pointer(d.fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(d.fn.fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(d.link)))) + } + for p := gp._panic; p != nil; p = p.link { + dumpint(tagPanic) + dumpint(uint64(uintptr(unsafe.Pointer(p)))) + dumpint(uint64(uintptr(unsafe.Pointer(gp)))) + eface := (*eface)(unsafe.Pointer(&p.arg)) + dumpint(uint64(uintptr(unsafe.Pointer(eface._type)))) + dumpint(uint64(uintptr(unsafe.Pointer(eface.data)))) + dumpint(0) // was p->defer, no longer recorded + dumpint(uint64(uintptr(unsafe.Pointer(p.link)))) + } +} + +func dumpgs() { + // goroutines & stacks + for i := 0; uintptr(i) < allglen; i++ { + gp := allgs[i] + status := readgstatus(gp) // The world is stopped so gp will not be in a scan state. + switch status { + default: + print("runtime: unexpected G.status ", hex(status), "\n") + gothrow("dumpgs in STW - bad status") + case _Gdead: + // ok + case _Grunnable, + _Gsyscall, + _Gwaiting: + dumpgoroutine(gp) + } + } +} + +func finq_callback(fn *funcval, obj unsafe.Pointer, nret uintptr, fint *_type, ot *ptrtype) { + dumpint(tagQueuedFinalizer) + dumpint(uint64(uintptr(obj))) + dumpint(uint64(uintptr(unsafe.Pointer(fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(fn.fn)))) + dumpint(uint64(uintptr(unsafe.Pointer(fint)))) + dumpint(uint64(uintptr(unsafe.Pointer(ot)))) +} + +func dumproots() { + // data segment + dumpbvtypes(&gcdatamask, unsafe.Pointer(&data)) + dumpint(tagData) + dumpint(uint64(uintptr(unsafe.Pointer(&data)))) + dumpmemrange(unsafe.Pointer(&data), uintptr(unsafe.Pointer(&edata))-uintptr(unsafe.Pointer(&data))) + dumpfields(gcdatamask) + + // bss segment + dumpbvtypes(&gcbssmask, unsafe.Pointer(&bss)) + dumpint(tagBSS) + dumpint(uint64(uintptr(unsafe.Pointer(&bss)))) + dumpmemrange(unsafe.Pointer(&bss), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss))) + dumpfields(gcbssmask) + + // MSpan.types + allspans := h_allspans + for spanidx := uint32(0); spanidx < mheap_.nspan; spanidx++ { + s := allspans[spanidx] + if s.state == _MSpanInUse { + // Finalizers + for sp := s.specials; sp != nil; sp = sp.next { + if sp.kind != _KindSpecialFinalizer { + continue + } + spf := (*specialfinalizer)(unsafe.Pointer(sp)) + p := unsafe.Pointer((uintptr(s.start) << _PageShift) + uintptr(spf.special.offset)) + dumpfinalizer(p, spf.fn, spf.fint, spf.ot) + } + } + } + + // Finalizer queue + iterate_finq(finq_callback) +} + +// Bit vector of free marks. +// Needs to be as big as the largest number of objects per span. +var freemark [_PageSize / 8]bool + +func dumpobjs() { + for i := uintptr(0); i < uintptr(mheap_.nspan); i++ { + s := h_allspans[i] + if s.state != _MSpanInUse { + continue + } + p := uintptr(s.start << _PageShift) + size := s.elemsize + n := (s.npages << _PageShift) / size + if n > uintptr(len(freemark)) { + gothrow("freemark array doesn't have enough entries") + } + for l := s.freelist; l != nil; l = l.next { + freemark[(uintptr(unsafe.Pointer(l))-p)/size] = true + } + for j := uintptr(0); j < n; j, p = j+1, p+size { + if freemark[j] { + freemark[j] = false + continue + } + dumpobj(unsafe.Pointer(p), size, makeheapobjbv(p, size)) + } + } +} + +func dumpparams() { + dumpint(tagParams) + x := uintptr(1) + if *(*byte)(unsafe.Pointer(&x)) == 1 { + dumpbool(false) // little-endian ptrs + } else { + dumpbool(true) // big-endian ptrs + } + dumpint(ptrSize) + dumpint(uint64(mheap_.arena_start)) + dumpint(uint64(mheap_.arena_used)) + dumpint(thechar) + dumpstr(goexperiment) + dumpint(uint64(ncpu)) +} + +func itab_callback(tab *itab) { + t := tab._type + // Dump a map from itab* to the type of its data field. + // We want this map so we can deduce types of interface referents. + if t.kind&kindDirectIface == 0 { + // indirect - data slot is a pointer to t. + dumptype(t.ptrto) + dumpint(tagItab) + dumpint(uint64(uintptr(unsafe.Pointer(tab)))) + dumpint(uint64(uintptr(unsafe.Pointer(t.ptrto)))) + } else if t.kind&kindNoPointers == 0 { + // t is pointer-like - data slot is a t. + dumptype(t) + dumpint(tagItab) + dumpint(uint64(uintptr(unsafe.Pointer(tab)))) + dumpint(uint64(uintptr(unsafe.Pointer(t)))) + } else { + // Data slot is a scalar. Dump type just for fun. + // With pointer-only interfaces, this shouldn't happen. + dumptype(t) + dumpint(tagItab) + dumpint(uint64(uintptr(unsafe.Pointer(tab)))) + dumpint(uint64(uintptr(unsafe.Pointer(t)))) + } +} + +func dumpitabs() { + iterate_itabs(itab_callback) +} + +func dumpms() { + for mp := allm; mp != nil; mp = mp.alllink { + dumpint(tagOSThread) + dumpint(uint64(uintptr(unsafe.Pointer(mp)))) + dumpint(uint64(mp.id)) + dumpint(mp.procid) + } +} + +func dumpmemstats() { + dumpint(tagMemStats) + dumpint(memstats.alloc) + dumpint(memstats.total_alloc) + dumpint(memstats.sys) + dumpint(memstats.nlookup) + dumpint(memstats.nmalloc) + dumpint(memstats.nfree) + dumpint(memstats.heap_alloc) + dumpint(memstats.heap_sys) + dumpint(memstats.heap_idle) + dumpint(memstats.heap_inuse) + dumpint(memstats.heap_released) + dumpint(memstats.heap_objects) + dumpint(memstats.stacks_inuse) + dumpint(memstats.stacks_sys) + dumpint(memstats.mspan_inuse) + dumpint(memstats.mspan_sys) + dumpint(memstats.mcache_inuse) + dumpint(memstats.mcache_sys) + dumpint(memstats.buckhash_sys) + dumpint(memstats.gc_sys) + dumpint(memstats.other_sys) + dumpint(memstats.next_gc) + dumpint(memstats.last_gc) + dumpint(memstats.pause_total_ns) + for i := 0; i < 256; i++ { + dumpint(memstats.pause_ns[i]) + } + dumpint(uint64(memstats.numgc)) +} + +func dumpmemprof_callback(b *bucket, nstk uintptr, pstk *uintptr, size, allocs, frees uintptr) { + stk := (*[100000]uintptr)(unsafe.Pointer(pstk)) + dumpint(tagMemProf) + dumpint(uint64(uintptr(unsafe.Pointer(b)))) + dumpint(uint64(size)) + dumpint(uint64(nstk)) + for i := uintptr(0); i < nstk; i++ { + pc := stk[i] + f := findfunc(pc) + if f == nil { + var buf [64]byte + n := len(buf) + n-- + buf[n] = ')' + if pc == 0 { + n-- + buf[n] = '0' + } else { + for pc > 0 { + n-- + buf[n] = "0123456789abcdef"[pc&15] + pc >>= 4 + } + } + n-- + buf[n] = 'x' + n-- + buf[n] = '0' + n-- + buf[n] = '(' + dumpslice(buf[n:]) + dumpstr("?") + dumpint(0) + } else { + dumpstr(gofuncname(f)) + if i > 0 && pc > f.entry { + pc-- + } + file, line := funcline(f, pc) + dumpstr(file) + dumpint(uint64(line)) + } + } + dumpint(uint64(allocs)) + dumpint(uint64(frees)) +} + +func dumpmemprof() { + iterate_memprof(dumpmemprof_callback) + allspans := h_allspans + for spanidx := uint32(0); spanidx < mheap_.nspan; spanidx++ { + s := allspans[spanidx] + if s.state != _MSpanInUse { + continue + } + for sp := s.specials; sp != nil; sp = sp.next { + if sp.kind != _KindSpecialProfile { + continue + } + spp := (*specialprofile)(unsafe.Pointer(sp)) + p := uintptr(s.start<<_PageShift) + uintptr(spp.special.offset) + dumpint(tagAllocSample) + dumpint(uint64(p)) + dumpint(uint64(uintptr(unsafe.Pointer(spp.b)))) + } + } +} + +var dumphdr = []byte("go1.4 heap dump\n") + +func mdump() { + // make sure we're done sweeping + for i := uintptr(0); i < uintptr(mheap_.nspan); i++ { + s := h_allspans[i] + if s.state == _MSpanInUse { + mSpan_EnsureSwept(s) + } + } + memclr(unsafe.Pointer(&typecache), unsafe.Sizeof(typecache)) + dwrite(unsafe.Pointer(&dumphdr[0]), uintptr(len(dumphdr))) + dumpparams() + dumpitabs() + dumpobjs() + dumpgs() + dumpms() + dumproots() + dumpmemstats() + dumpmemprof() + dumpint(tagEOF) + flush() +} + +func writeheapdump_m(fd uintptr) { + _g_ := getg() + casgstatus(_g_.m.curg, _Grunning, _Gwaiting) + _g_.waitreason = "dumping heap" + + // Update stats so we can dump them. + // As a side effect, flushes all the MCaches so the MSpan.freelist + // lists contain all the free objects. + updatememstats(nil) + + // Set dump file. + dumpfd = fd + + // Call dump routine. + mdump() + + // Reset dump file. + dumpfd = 0 + if tmpbuf != nil { + sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys) + tmpbuf = nil + } + + casgstatus(_g_.m.curg, _Gwaiting, _Grunning) +} + +// dumpint() the kind & offset of each field in an object. +func dumpfields(bv bitvector) { + dumpbv(&bv, 0) + dumpint(fieldKindEol) +} + +// The heap dump reader needs to be able to disambiguate +// Eface entries. So it needs to know every type that might +// appear in such an entry. The following routine accomplishes that. +// TODO(rsc, khr): Delete - no longer possible. + +// Dump all the types that appear in the type field of +// any Eface described by this bit vector. +func dumpbvtypes(bv *bitvector, base unsafe.Pointer) { +} + +func makeheapobjbv(p uintptr, size uintptr) bitvector { + // Extend the temp buffer if necessary. + nptr := size / ptrSize + if uintptr(len(tmpbuf)) < nptr*_BitsPerPointer/8+1 { + if tmpbuf != nil { + sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys) + } + n := nptr*_BitsPerPointer/8 + 1 + p := sysAlloc(n, &memstats.other_sys) + if p == nil { + gothrow("heapdump: out of memory") + } + tmpbuf = (*[1 << 30]byte)(p)[:n] + } + // Copy and compact the bitmap. + var i uintptr + for i = 0; i < nptr; i++ { + off := (p + i*ptrSize - mheap_.arena_start) / ptrSize + bitp := (*uint8)(unsafe.Pointer(mheap_.arena_start - off/wordsPerBitmapByte - 1)) + shift := uint8((off % wordsPerBitmapByte) * gcBits) + bits := (*bitp >> (shift + 2)) & _BitsMask + if bits == _BitsDead { + break // end of heap object + } + tmpbuf[i*_BitsPerPointer/8] &^= (_BitsMask << ((i * _BitsPerPointer) % 8)) + tmpbuf[i*_BitsPerPointer/8] |= bits << ((i * _BitsPerPointer) % 8) + } + return bitvector{int32(i * _BitsPerPointer), &tmpbuf[0]} +} diff --git a/src/runtime/lfstack.c b/src/runtime/lfstack.c deleted file mode 100644 index 57e0af282..000000000 --- a/src/runtime/lfstack.c +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Lock-free stack. -// The following code runs only on g0 stack. - -#include "runtime.h" -#include "arch_GOARCH.h" - -#ifdef _64BIT -// Amd64 uses 48-bit virtual addresses, 47-th bit is used as kernel/user flag. -// So we use 17msb of pointers as ABA counter. -# define PTR_BITS 47 -#else -# define PTR_BITS 32 -#endif -#define PTR_MASK ((1ull<<PTR_BITS)-1) -#define CNT_MASK (0ull-1) - -#ifdef _64BIT -#ifdef GOOS_solaris -// SPARC64 and Solaris on AMD64 uses all 64 bits of virtual addresses. -// Use low-order three bits as ABA counter. -// http://docs.oracle.com/cd/E19120-01/open.solaris/816-5138/6mba6ua5p/index.html -#undef PTR_BITS -#undef CNT_MASK -#undef PTR_MASK -#define PTR_BITS 0 -#define CNT_MASK 7 -#define PTR_MASK ((0ull-1)<<3) -#endif -#endif - -void -runtime·lfstackpush(uint64 *head, LFNode *node) -{ - uint64 old, new; - - if((uintptr)node != ((uintptr)node&PTR_MASK)) { - runtime·printf("p=%p\n", node); - runtime·throw("runtime·lfstackpush: invalid pointer"); - } - - node->pushcnt++; - new = (uint64)(uintptr)node|(((uint64)node->pushcnt&CNT_MASK)<<PTR_BITS); - for(;;) { - old = runtime·atomicload64(head); - node->next = (LFNode*)(uintptr)(old&PTR_MASK); - if(runtime·cas64(head, old, new)) - break; - } -} - -LFNode* -runtime·lfstackpop(uint64 *head) -{ - LFNode *node, *node2; - uint64 old, new; - - for(;;) { - old = runtime·atomicload64(head); - if(old == 0) - return nil; - node = (LFNode*)(uintptr)(old&PTR_MASK); - node2 = runtime·atomicloadp(&node->next); - new = 0; - if(node2 != nil) - new = (uint64)(uintptr)node2|(((uint64)node2->pushcnt&CNT_MASK)<<PTR_BITS); - if(runtime·cas64(head, old, new)) - return node; - } -} - -void -runtime·lfstackpush_m(void) -{ - runtime·lfstackpush(g->m->ptrarg[0], g->m->ptrarg[1]); - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; -} - -void -runtime·lfstackpop_m(void) -{ - g->m->ptrarg[0] = runtime·lfstackpop(g->m->ptrarg[0]); -} diff --git a/src/runtime/lfstack.go b/src/runtime/lfstack.go new file mode 100644 index 000000000..8a36a67b3 --- /dev/null +++ b/src/runtime/lfstack.go @@ -0,0 +1,44 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Lock-free stack. +// The following code runs only on g0 stack. + +package runtime + +import "unsafe" + +func lfstackpush(head *uint64, node *lfnode) { + node.pushcnt++ + new := lfstackPack(node, node.pushcnt) + if node1, _ := lfstackUnpack(new); node1 != node { + println("runtime: lfstackpush invalid packing: node=", node, " cnt=", hex(node.pushcnt), " packed=", hex(new), " -> node=", node1, "\n") + gothrow("lfstackpush") + } + for { + old := atomicload64(head) + node.next, _ = lfstackUnpack(old) + if cas64(head, old, new) { + break + } + } +} + +func lfstackpop(head *uint64) unsafe.Pointer { + for { + old := atomicload64(head) + if old == 0 { + return nil + } + node, _ := lfstackUnpack(old) + node2 := (*lfnode)(atomicloadp(unsafe.Pointer(&node.next))) + new := uint64(0) + if node2 != nil { + new = lfstackPack(node2, node2.pushcnt) + } + if cas64(head, old, new) { + return unsafe.Pointer(node) + } + } +} diff --git a/src/runtime/lfstack_32bit.go b/src/runtime/lfstack_32bit.go new file mode 100644 index 000000000..61d8678d9 --- /dev/null +++ b/src/runtime/lfstack_32bit.go @@ -0,0 +1,21 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 arm + +package runtime + +import "unsafe" + +// On 32-bit systems, the stored uint64 has a 32-bit pointer and 32-bit count. + +func lfstackPack(node *lfnode, cnt uintptr) uint64 { + return uint64(uintptr(unsafe.Pointer(node)))<<32 | uint64(cnt) +} + +func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) { + node = (*lfnode)(unsafe.Pointer(uintptr(val >> 32))) + cnt = uintptr(val) + return +} diff --git a/src/runtime/lfstack_amd64.go b/src/runtime/lfstack_amd64.go new file mode 100644 index 000000000..84e28519f --- /dev/null +++ b/src/runtime/lfstack_amd64.go @@ -0,0 +1,24 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// On AMD64, virtual addresses are 48-bit numbers sign extended to 64. +// We shift the address left 16 to eliminate the sign extended part and make +// room in the bottom for the count. +// In addition to the 16 bits taken from the top, we can take 3 from the +// bottom, because node must be pointer-aligned, giving a total of 19 bits +// of count. + +func lfstackPack(node *lfnode, cnt uintptr) uint64 { + return uint64(uintptr(unsafe.Pointer(node)))<<16 | uint64(cnt&(1<<19-1)) +} + +func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) { + node = (*lfnode)(unsafe.Pointer(uintptr(int64(val) >> 19 << 3))) + cnt = uintptr(val & (1<<19 - 1)) + return +} diff --git a/src/runtime/lfstack_linux_power64x.go b/src/runtime/lfstack_linux_power64x.go new file mode 100644 index 000000000..89e389fc7 --- /dev/null +++ b/src/runtime/lfstack_linux_power64x.go @@ -0,0 +1,32 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le +// +build linux + +package runtime + +import "unsafe" + +// On Power64, Linux limits the user address space to 46 bits (see +// TASK_SIZE_USER64 in the Linux kernel). This has grown over time, +// so here we allow 48 bit addresses. +// +// In addition to the 16 bits taken from the top, we can take 3 from the +// bottom, because node must be pointer-aligned, giving a total of 19 bits +// of count. +const ( + addrBits = 48 + cntBits = 64 - addrBits + 3 +) + +func lfstackPack(node *lfnode, cnt uintptr) uint64 { + return uint64(uintptr(unsafe.Pointer(node)))<<(64-addrBits) | uint64(cnt&(1<<cntBits-1)) +} + +func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) { + node = (*lfnode)(unsafe.Pointer(uintptr(val >> cntBits << 3))) + cnt = uintptr(val & (1<<cntBits - 1)) + return +} diff --git a/src/runtime/lock_futex.go b/src/runtime/lock_futex.go index 725962341..11c3a3f06 100644 --- a/src/runtime/lock_futex.go +++ b/src/runtime/lock_futex.go @@ -34,9 +34,6 @@ const ( // Note that there can be spinning threads during all states - they do not // affect mutex's state. -func futexsleep(addr *uint32, val uint32, ns int64) -func futexwakeup(addr *uint32, cnt uint32) - // We use the uintptr mutex.key and note.key as a uint32. func key32(p *uintptr) *uint32 { return (*uint32)(unsafe.Pointer(p)) @@ -198,8 +195,8 @@ func notetsleepg(n *note, ns int64) bool { gothrow("notetsleepg on g0") } - entersyscallblock() + entersyscallblock(0) ok := notetsleep_internal(n, ns) - exitsyscall() + exitsyscall(0) return ok } diff --git a/src/runtime/lock_sema.go b/src/runtime/lock_sema.go index d136b8280..a2a87bac4 100644 --- a/src/runtime/lock_sema.go +++ b/src/runtime/lock_sema.go @@ -31,10 +31,6 @@ const ( passive_spin = 1 ) -func semacreate() uintptr -func semasleep(int64) int32 -func semawakeup(mp *m) - func lock(l *mutex) { gp := getg() if gp.m.locks < 0 { @@ -263,8 +259,8 @@ func notetsleepg(n *note, ns int64) bool { if gp.m.waitsema == 0 { gp.m.waitsema = semacreate() } - entersyscallblock() + entersyscallblock(0) ok := notetsleep_internal(n, ns, nil, 0) - exitsyscall() + exitsyscall(0) return ok } diff --git a/src/runtime/malloc.c b/src/runtime/malloc.c deleted file mode 100644 index b79c30b72..000000000 --- a/src/runtime/malloc.c +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// See malloc.h for overview. -// -// TODO(rsc): double-check stats. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "type.h" -#include "typekind.h" -#include "race.h" -#include "stack.h" -#include "textflag.h" - -// Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K. -#pragma dataflag NOPTR -MHeap runtime·mheap; -#pragma dataflag NOPTR -MStats runtime·memstats; - -int32 -runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) -{ - uintptr n, i; - byte *p; - MSpan *s; - - g->m->mcache->local_nlookup++; - if (sizeof(void*) == 4 && g->m->mcache->local_nlookup >= (1<<30)) { - // purge cache stats to prevent overflow - runtime·lock(&runtime·mheap.lock); - runtime·purgecachedstats(g->m->mcache); - runtime·unlock(&runtime·mheap.lock); - } - - s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); - if(sp) - *sp = s; - if(s == nil) { - if(base) - *base = nil; - if(size) - *size = 0; - return 0; - } - - p = (byte*)((uintptr)s->start<<PageShift); - if(s->sizeclass == 0) { - // Large object. - if(base) - *base = p; - if(size) - *size = s->npages<<PageShift; - return 1; - } - - n = s->elemsize; - if(base) { - i = ((byte*)v - p)/n; - *base = p + i*n; - } - if(size) - *size = n; - - return 1; -} - -#pragma textflag NOSPLIT -void -runtime·purgecachedstats(MCache *c) -{ - MHeap *h; - int32 i; - - // Protected by either heap or GC lock. - h = &runtime·mheap; - mstats.heap_alloc += c->local_cachealloc; - c->local_cachealloc = 0; - mstats.tinyallocs += c->local_tinyallocs; - c->local_tinyallocs = 0; - mstats.nlookup += c->local_nlookup; - c->local_nlookup = 0; - h->largefree += c->local_largefree; - c->local_largefree = 0; - h->nlargefree += c->local_nlargefree; - c->local_nlargefree = 0; - for(i=0; i<nelem(c->local_nsmallfree); i++) { - h->nsmallfree[i] += c->local_nsmallfree[i]; - c->local_nsmallfree[i] = 0; - } -} - -// Size of the trailing by_size array differs between Go and C, -// and all data after by_size is local to C, not exported to Go. -// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. -// sizeof_C_MStats is what C thinks about size of Go struct. -uintptr runtime·sizeof_C_MStats = offsetof(MStats, by_size[61]); - -#define MaxArena32 (2U<<30) - -// For use by Go. If it were a C enum it would be made available automatically, -// but the value of MaxMem is too large for enum. -uintptr runtime·maxmem = MaxMem; - -void -runtime·mallocinit(void) -{ - byte *p, *p1; - uintptr arena_size, bitmap_size, spans_size, p_size; - extern byte runtime·end[]; - uintptr limit; - uint64 i; - bool reserved; - - p = nil; - p_size = 0; - arena_size = 0; - bitmap_size = 0; - spans_size = 0; - reserved = false; - - // for 64-bit build - USED(p); - USED(p_size); - USED(arena_size); - USED(bitmap_size); - USED(spans_size); - - runtime·InitSizes(); - - if(runtime·class_to_size[TinySizeClass] != TinySize) - runtime·throw("bad TinySizeClass"); - - // limit = runtime·memlimit(); - // See https://code.google.com/p/go/issues/detail?id=5049 - // TODO(rsc): Fix after 1.1. - limit = 0; - - // Set up the allocation arena, a contiguous area of memory where - // allocated data will be found. The arena begins with a bitmap large - // enough to hold 4 bits per allocated word. - if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { - // On a 64-bit machine, allocate from a single contiguous reservation. - // 128 GB (MaxMem) should be big enough for now. - // - // The code will work with the reservation at any address, but ask - // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). - // Allocating a 128 GB region takes away 37 bits, and the amd64 - // doesn't let us choose the top 17 bits, so that leaves the 11 bits - // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means - // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. - // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid - // UTF-8 sequences, and they are otherwise as far away from - // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 - // addresses. An earlier attempt to use 0x11f8 caused out of memory errors - // on OS X during thread allocations. 0x00c0 causes conflicts with - // AddressSanitizer which reserves all memory up to 0x0100. - // These choices are both for debuggability and to reduce the - // odds of the conservative garbage collector not collecting memory - // because some non-pointer block of memory had a bit pattern - // that matched a memory address. - // - // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) - // but it hardly matters: e0 00 is not valid UTF-8 either. - // - // If this fails we fall back to the 32 bit memory mechanism - arena_size = MaxMem; - bitmap_size = arena_size / (sizeof(void*)*8/4); - spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); - spans_size = ROUND(spans_size, PageSize); - for(i = 0; i <= 0x7f; i++) { - p = (void*)(i<<40 | 0x00c0ULL<<32); - p_size = bitmap_size + spans_size + arena_size + PageSize; - p = runtime·SysReserve(p, p_size, &reserved); - if(p != nil) - break; - } - } - if (p == nil) { - // On a 32-bit machine, we can't typically get away - // with a giant virtual address space reservation. - // Instead we map the memory information bitmap - // immediately after the data segment, large enough - // to handle another 2GB of mappings (256 MB), - // along with a reservation for another 512 MB of memory. - // When that gets used up, we'll start asking the kernel - // for any memory anywhere and hope it's in the 2GB - // following the bitmap (presumably the executable begins - // near the bottom of memory, so we'll have to use up - // most of memory before the kernel resorts to giving out - // memory before the beginning of the text segment). - // - // Alternatively we could reserve 512 MB bitmap, enough - // for 4GB of mappings, and then accept any memory the - // kernel threw at us, but normally that's a waste of 512 MB - // of address space, which is probably too much in a 32-bit world. - bitmap_size = MaxArena32 / (sizeof(void*)*8/4); - arena_size = 512<<20; - spans_size = MaxArena32 / PageSize * sizeof(runtime·mheap.spans[0]); - if(limit > 0 && arena_size+bitmap_size+spans_size > limit) { - bitmap_size = (limit / 9) & ~((1<<PageShift) - 1); - arena_size = bitmap_size * 8; - spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); - } - spans_size = ROUND(spans_size, PageSize); - - // SysReserve treats the address we ask for, end, as a hint, - // not as an absolute requirement. If we ask for the end - // of the data segment but the operating system requires - // a little more space before we can start allocating, it will - // give out a slightly higher pointer. Except QEMU, which - // is buggy, as usual: it won't adjust the pointer upward. - // So adjust it upward a little bit ourselves: 1/4 MB to get - // away from the running binary image and then round up - // to a MB boundary. - p = (byte*)ROUND((uintptr)runtime·end + (1<<18), 1<<20); - p_size = bitmap_size + spans_size + arena_size + PageSize; - p = runtime·SysReserve(p, p_size, &reserved); - if(p == nil) - runtime·throw("runtime: cannot reserve arena virtual address space"); - } - - // PageSize can be larger than OS definition of page size, - // so SysReserve can give us a PageSize-unaligned pointer. - // To overcome this we ask for PageSize more and round up the pointer. - p1 = (byte*)ROUND((uintptr)p, PageSize); - - runtime·mheap.spans = (MSpan**)p1; - runtime·mheap.bitmap = p1 + spans_size; - runtime·mheap.arena_start = p1 + spans_size + bitmap_size; - runtime·mheap.arena_used = runtime·mheap.arena_start; - runtime·mheap.arena_end = p + p_size; - runtime·mheap.arena_reserved = reserved; - - if(((uintptr)runtime·mheap.arena_start & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in mallocinit"); - - // Initialize the rest of the allocator. - runtime·MHeap_Init(&runtime·mheap); - g->m->mcache = runtime·allocmcache(); -} - -void* -runtime·MHeap_SysAlloc(MHeap *h, uintptr n) -{ - byte *p, *p_end; - uintptr p_size; - bool reserved; - - if(n > h->arena_end - h->arena_used) { - // We are in 32-bit mode, maybe we didn't use all possible address space yet. - // Reserve some more space. - byte *new_end; - - p_size = ROUND(n + PageSize, 256<<20); - new_end = h->arena_end + p_size; - if(new_end <= h->arena_start + MaxArena32) { - // TODO: It would be bad if part of the arena - // is reserved and part is not. - p = runtime·SysReserve(h->arena_end, p_size, &reserved); - if(p == h->arena_end) { - h->arena_end = new_end; - h->arena_reserved = reserved; - } - else if(p+p_size <= h->arena_start + MaxArena32) { - // Keep everything page-aligned. - // Our pages are bigger than hardware pages. - h->arena_end = p+p_size; - h->arena_used = p + (-(uintptr)p&(PageSize-1)); - h->arena_reserved = reserved; - } else { - uint64 stat; - stat = 0; - runtime·SysFree(p, p_size, &stat); - } - } - } - if(n <= h->arena_end - h->arena_used) { - // Keep taking from our reservation. - p = h->arena_used; - runtime·SysMap(p, n, h->arena_reserved, &mstats.heap_sys); - h->arena_used += n; - runtime·MHeap_MapBits(h); - runtime·MHeap_MapSpans(h); - if(raceenabled) - runtime·racemapshadow(p, n); - - if(((uintptr)p & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in MHeap_SysAlloc"); - return p; - } - - // If using 64-bit, our reservation is all we have. - if(h->arena_end - h->arena_start >= MaxArena32) - return nil; - - // On 32-bit, once the reservation is gone we can - // try to get memory at a location chosen by the OS - // and hope that it is in the range we allocated bitmap for. - p_size = ROUND(n, PageSize) + PageSize; - p = runtime·sysAlloc(p_size, &mstats.heap_sys); - if(p == nil) - return nil; - - if(p < h->arena_start || p+p_size - h->arena_start >= MaxArena32) { - runtime·printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n", - p, h->arena_start, h->arena_start+MaxArena32); - runtime·SysFree(p, p_size, &mstats.heap_sys); - return nil; - } - - p_end = p + p_size; - p += -(uintptr)p & (PageSize-1); - if(p+n > h->arena_used) { - h->arena_used = p+n; - if(p_end > h->arena_end) - h->arena_end = p_end; - runtime·MHeap_MapBits(h); - runtime·MHeap_MapSpans(h); - if(raceenabled) - runtime·racemapshadow(p, n); - } - - if(((uintptr)p & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in MHeap_SysAlloc"); - return p; -} - -void -runtime·setFinalizer_m(void) -{ - FuncVal *fn; - void *arg; - uintptr nret; - Type *fint; - PtrType *ot; - - fn = g->m->ptrarg[0]; - arg = g->m->ptrarg[1]; - nret = g->m->scalararg[0]; - fint = g->m->ptrarg[2]; - ot = g->m->ptrarg[3]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - g->m->ptrarg[2] = nil; - g->m->ptrarg[3] = nil; - - g->m->scalararg[0] = runtime·addfinalizer(arg, fn, nret, fint, ot); -} - -void -runtime·removeFinalizer_m(void) -{ - void *p; - - p = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - runtime·removefinalizer(p); -} - -// mcallable cache refill -void -runtime·mcacheRefill_m(void) -{ - runtime·MCache_Refill(g->m->mcache, (int32)g->m->scalararg[0]); -} - -void -runtime·largeAlloc_m(void) -{ - uintptr npages, size; - MSpan *s; - void *v; - int32 flag; - - //runtime·printf("largeAlloc size=%D\n", g->m->scalararg[0]); - // Allocate directly from heap. - size = g->m->scalararg[0]; - flag = (int32)g->m->scalararg[1]; - if(size + PageSize < size) - runtime·throw("out of memory"); - npages = size >> PageShift; - if((size & PageMask) != 0) - npages++; - s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZero)); - if(s == nil) - runtime·throw("out of memory"); - s->limit = (byte*)(s->start<<PageShift) + size; - v = (void*)(s->start << PageShift); - // setup for mark sweep - runtime·markspan(v, 0, 0, true); - g->m->ptrarg[0] = s; -} diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 117044944..d73d1ba6a 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -4,9 +4,7 @@ package runtime -import ( - "unsafe" -) +import "unsafe" const ( debugMalloc = false @@ -28,10 +26,11 @@ const ( maxGCMask = _MaxGCMask bitsDead = _BitsDead bitsPointer = _BitsPointer + bitsScalar = _BitsScalar mSpanInUse = _MSpanInUse - concurrentSweep = _ConcurrentSweep != 0 + concurrentSweep = _ConcurrentSweep ) // Page number (address>>pageShift) @@ -56,7 +55,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // This function must be atomic wrt GC, but for performance reasons // we don't acquirem/releasem on fast path. The code below does not have // split stack checks, so it can't be preempted by GC. - // Functions like roundup/add are inlined. And onM/racemalloc are nosplit. + // Functions like roundup/add are inlined. And systemstack/racemalloc are nosplit. // If debugMalloc = true, these assumptions are checked below. if debugMalloc { mp := acquirem() @@ -142,10 +141,9 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { s = c.alloc[tinySizeClass] v := s.freelist if v == nil { - mp := acquirem() - mp.scalararg[0] = tinySizeClass - onM(mcacheRefill_m) - releasem(mp) + systemstack(func() { + mCache_Refill(c, tinySizeClass) + }) s = c.alloc[tinySizeClass] v = s.freelist } @@ -173,10 +171,9 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { s = c.alloc[sizeclass] v := s.freelist if v == nil { - mp := acquirem() - mp.scalararg[0] = uintptr(sizeclass) - onM(mcacheRefill_m) - releasem(mp) + systemstack(func() { + mCache_Refill(c, int32(sizeclass)) + }) s = c.alloc[sizeclass] v = s.freelist } @@ -193,13 +190,10 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { } c.local_cachealloc += intptr(size) } else { - mp := acquirem() - mp.scalararg[0] = uintptr(size) - mp.scalararg[1] = uintptr(flags) - onM(largeAlloc_m) - s = (*mspan)(mp.ptrarg[0]) - mp.ptrarg[0] = nil - releasem(mp) + var s *mspan + systemstack(func() { + s = largeAlloc(size, uint32(flags)) + }) x = unsafe.Pointer(uintptr(s.start << pageShift)) size = uintptr(s.elemsize) } @@ -251,22 +245,19 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // into the GC bitmap. It's 7 times slower than copying // from the pre-unrolled mask, but saves 1/16 of type size // memory for the mask. - mp := acquirem() - mp.ptrarg[0] = x - mp.ptrarg[1] = unsafe.Pointer(typ) - mp.scalararg[0] = uintptr(size) - mp.scalararg[1] = uintptr(size0) - onM(unrollgcproginplace_m) - releasem(mp) + systemstack(func() { + unrollgcproginplace_m(x, typ, size, size0) + }) goto marked } ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) - // Check whether the program is already unrolled. - if uintptr(atomicloadp(unsafe.Pointer(ptrmask)))&0xff == 0 { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(typ) - onM(unrollgcprog_m) - releasem(mp) + // Check whether the program is already unrolled + // by checking if the unroll flag byte is set + maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask))) + if *(*uint8)(unsafe.Pointer(&maskword)) == 0 { + systemstack(func() { + unrollgcprog_m(typ) + }) } ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte } else { @@ -359,7 +350,7 @@ func newarray(typ *_type, n uintptr) unsafe.Pointer { if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } - if int(n) < 0 || (typ.size > 0 && n > maxmem/uintptr(typ.size)) { + if int(n) < 0 || (typ.size > 0 && n > _MaxMem/uintptr(typ.size)) { panic("runtime: allocation size out of range") } return mallocgc(uintptr(typ.size)*n, typ, flags) @@ -438,7 +429,7 @@ func gogc(force int32) { mp = acquirem() mp.gcing = 1 releasem(mp) - onM(stoptheworld) + systemstack(stoptheworld) if mp != acquirem() { gothrow("gogc: rescheduled") } @@ -459,20 +450,16 @@ func gogc(force int32) { startTime = nanotime() } // switch to g0, call gc, then switch back - mp.scalararg[0] = uintptr(uint32(startTime)) // low 32 bits - mp.scalararg[1] = uintptr(startTime >> 32) // high 32 bits - if force >= 2 { - mp.scalararg[2] = 1 // eagersweep - } else { - mp.scalararg[2] = 0 - } - onM(gc_m) + eagersweep := force >= 2 + systemstack(func() { + gc_m(startTime, eagersweep) + }) } // all done mp.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) releasem(mp) mp = nil @@ -591,11 +578,10 @@ func SetFinalizer(obj interface{}, finalizer interface{}) { f := (*eface)(unsafe.Pointer(&finalizer)) ftyp := f._type if ftyp == nil { - // switch to M stack and remove finalizer - mp := acquirem() - mp.ptrarg[0] = e.data - onM(removeFinalizer_m) - releasem(mp) + // switch to system stack and remove finalizer + systemstack(func() { + removefinalizer(e.data) + }) return } @@ -640,18 +626,11 @@ okarg: // make sure we have a finalizer goroutine createfing() - // switch to M stack to add finalizer record - mp := acquirem() - mp.ptrarg[0] = f.data - mp.ptrarg[1] = e.data - mp.scalararg[0] = nret - mp.ptrarg[2] = unsafe.Pointer(fint) - mp.ptrarg[3] = unsafe.Pointer(ot) - onM(setFinalizer_m) - if mp.scalararg[0] != 1 { - gothrow("runtime.SetFinalizer: finalizer already set") - } - releasem(mp) + systemstack(func() { + if !addfinalizer(e.data, (*funcval)(f.data), nret, fint, ot) { + gothrow("runtime.SetFinalizer: finalizer already set") + } + }) } // round n up to a multiple of a. a must be a power of 2. diff --git a/src/runtime/malloc.h b/src/runtime/malloc.h deleted file mode 100644 index adb8d3d67..000000000 --- a/src/runtime/malloc.h +++ /dev/null @@ -1,621 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Memory allocator, based on tcmalloc. -// http://goog-perftools.sourceforge.net/doc/tcmalloc.html - -// The main allocator works in runs of pages. -// Small allocation sizes (up to and including 32 kB) are -// rounded to one of about 100 size classes, each of which -// has its own free list of objects of exactly that size. -// Any free page of memory can be split into a set of objects -// of one size class, which are then managed using free list -// allocators. -// -// The allocator's data structures are: -// -// FixAlloc: a free-list allocator for fixed-size objects, -// used to manage storage used by the allocator. -// MHeap: the malloc heap, managed at page (4096-byte) granularity. -// MSpan: a run of pages managed by the MHeap. -// MCentral: a shared free list for a given size class. -// MCache: a per-thread (in Go, per-P) cache for small objects. -// MStats: allocation statistics. -// -// Allocating a small object proceeds up a hierarchy of caches: -// -// 1. Round the size up to one of the small size classes -// and look in the corresponding MCache free list. -// If the list is not empty, allocate an object from it. -// This can all be done without acquiring a lock. -// -// 2. If the MCache free list is empty, replenish it by -// taking a bunch of objects from the MCentral free list. -// Moving a bunch amortizes the cost of acquiring the MCentral lock. -// -// 3. If the MCentral free list is empty, replenish it by -// allocating a run of pages from the MHeap and then -// chopping that memory into a objects of the given size. -// Allocating many objects amortizes the cost of locking -// the heap. -// -// 4. If the MHeap is empty or has no page runs large enough, -// allocate a new group of pages (at least 1MB) from the -// operating system. Allocating a large run of pages -// amortizes the cost of talking to the operating system. -// -// Freeing a small object proceeds up the same hierarchy: -// -// 1. Look up the size class for the object and add it to -// the MCache free list. -// -// 2. If the MCache free list is too long or the MCache has -// too much memory, return some to the MCentral free lists. -// -// 3. If all the objects in a given span have returned to -// the MCentral list, return that span to the page heap. -// -// 4. If the heap has too much memory, return some to the -// operating system. -// -// TODO(rsc): Step 4 is not implemented. -// -// Allocating and freeing a large object uses the page heap -// directly, bypassing the MCache and MCentral free lists. -// -// The small objects on the MCache and MCentral free lists -// may or may not be zeroed. They are zeroed if and only if -// the second word of the object is zero. A span in the -// page heap is zeroed unless s->needzero is set. When a span -// is allocated to break into small objects, it is zeroed if needed -// and s->needzero is set. There are two main benefits to delaying the -// zeroing this way: -// -// 1. stack frames allocated from the small object lists -// or the page heap can avoid zeroing altogether. -// 2. the cost of zeroing when reusing a small object is -// charged to the mutator, not the garbage collector. -// -// This C code was written with an eye toward translating to Go -// in the future. Methods have the form Type_Method(Type *t, ...). - -typedef struct MCentral MCentral; -typedef struct MHeap MHeap; -typedef struct MSpan MSpan; -typedef struct MStats MStats; -typedef struct MLink MLink; -typedef struct GCStats GCStats; - -enum -{ - PageShift = 13, - PageSize = 1<<PageShift, - PageMask = PageSize - 1, -}; -typedef uintptr pageID; // address >> PageShift - -enum -{ - // Computed constant. The definition of MaxSmallSize and the - // algorithm in msize.c produce some number of different allocation - // size classes. NumSizeClasses is that number. It's needed here - // because there are static arrays of this length; when msize runs its - // size choosing algorithm it double-checks that NumSizeClasses agrees. - NumSizeClasses = 67, - - // Tunable constants. - MaxSmallSize = 32<<10, - - // Tiny allocator parameters, see "Tiny allocator" comment in malloc.goc. - TinySize = 16, - TinySizeClass = 2, - - FixAllocChunk = 16<<10, // Chunk size for FixAlloc - MaxMHeapList = 1<<(20 - PageShift), // Maximum page length for fixed-size list in MHeap. - HeapAllocChunk = 1<<20, // Chunk size for heap growth - - // Per-P, per order stack segment cache size. - StackCacheSize = 32*1024, - // Number of orders that get caching. Order 0 is FixedStack - // and each successive order is twice as large. - NumStackOrders = 3, - - // Number of bits in page to span calculations (4k pages). - // On Windows 64-bit we limit the arena to 32GB or 35 bits (see below for reason). - // On other 64-bit platforms, we limit the arena to 128GB, or 37 bits. - // On 32-bit, we don't bother limiting anything, so we use the full 32-bit address. -#ifdef _64BIT -#ifdef GOOS_windows - // Windows counts memory used by page table into committed memory - // of the process, so we can't reserve too much memory. - // See http://golang.org/issue/5402 and http://golang.org/issue/5236. - MHeapMap_Bits = 35 - PageShift, -#else - MHeapMap_Bits = 37 - PageShift, -#endif -#else - MHeapMap_Bits = 32 - PageShift, -#endif - - // Max number of threads to run garbage collection. - // 2, 3, and 4 are all plausible maximums depending - // on the hardware details of the machine. The garbage - // collector scales well to 32 cpus. - MaxGcproc = 32, -}; - -// Maximum memory allocation size, a hint for callers. -// This must be a #define instead of an enum because it -// is so large. -#ifdef _64BIT -#define MaxMem (1ULL<<(MHeapMap_Bits+PageShift)) /* 128 GB or 32 GB */ -#else -#define MaxMem ((uintptr)-1) -#endif - -// A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) -struct MLink -{ - MLink *next; -}; - -// sysAlloc obtains a large chunk of zeroed memory from the -// operating system, typically on the order of a hundred kilobytes -// or a megabyte. -// NOTE: sysAlloc returns OS-aligned memory, but the heap allocator -// may use larger alignment, so the caller must be careful to realign the -// memory obtained by sysAlloc. -// -// SysUnused notifies the operating system that the contents -// of the memory region are no longer needed and can be reused -// for other purposes. -// SysUsed notifies the operating system that the contents -// of the memory region are needed again. -// -// SysFree returns it unconditionally; this is only used if -// an out-of-memory error has been detected midway through -// an allocation. It is okay if SysFree is a no-op. -// -// SysReserve reserves address space without allocating memory. -// If the pointer passed to it is non-nil, the caller wants the -// reservation there, but SysReserve can still choose another -// location if that one is unavailable. On some systems and in some -// cases SysReserve will simply check that the address space is -// available and not actually reserve it. If SysReserve returns -// non-nil, it sets *reserved to true if the address space is -// reserved, false if it has merely been checked. -// NOTE: SysReserve returns OS-aligned memory, but the heap allocator -// may use larger alignment, so the caller must be careful to realign the -// memory obtained by sysAlloc. -// -// SysMap maps previously reserved address space for use. -// The reserved argument is true if the address space was really -// reserved, not merely checked. -// -// SysFault marks a (already sysAlloc'd) region to fault -// if accessed. Used only for debugging the runtime. - -void* runtime·sysAlloc(uintptr nbytes, uint64 *stat); -void runtime·SysFree(void *v, uintptr nbytes, uint64 *stat); -void runtime·SysUnused(void *v, uintptr nbytes); -void runtime·SysUsed(void *v, uintptr nbytes); -void runtime·SysMap(void *v, uintptr nbytes, bool reserved, uint64 *stat); -void* runtime·SysReserve(void *v, uintptr nbytes, bool *reserved); -void runtime·SysFault(void *v, uintptr nbytes); - -// FixAlloc is a simple free-list allocator for fixed size objects. -// Malloc uses a FixAlloc wrapped around sysAlloc to manages its -// MCache and MSpan objects. -// -// Memory returned by FixAlloc_Alloc is not zeroed. -// The caller is responsible for locking around FixAlloc calls. -// Callers can keep state in the object but the first word is -// smashed by freeing and reallocating. -struct FixAlloc -{ - uintptr size; - void (*first)(void *arg, byte *p); // called first time p is returned - void* arg; - MLink* list; - byte* chunk; - uint32 nchunk; - uintptr inuse; // in-use bytes now - uint64* stat; -}; - -void runtime·FixAlloc_Init(FixAlloc *f, uintptr size, void (*first)(void*, byte*), void *arg, uint64 *stat); -void* runtime·FixAlloc_Alloc(FixAlloc *f); -void runtime·FixAlloc_Free(FixAlloc *f, void *p); - - -// Statistics. -// Shared with Go: if you edit this structure, also edit type MemStats in mem.go. -struct MStats -{ - // General statistics. - uint64 alloc; // bytes allocated and still in use - uint64 total_alloc; // bytes allocated (even if freed) - uint64 sys; // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) - uint64 nlookup; // number of pointer lookups - uint64 nmalloc; // number of mallocs - uint64 nfree; // number of frees - - // Statistics about malloc heap. - // protected by mheap.lock - uint64 heap_alloc; // bytes allocated and still in use - uint64 heap_sys; // bytes obtained from system - uint64 heap_idle; // bytes in idle spans - uint64 heap_inuse; // bytes in non-idle spans - uint64 heap_released; // bytes released to the OS - uint64 heap_objects; // total number of allocated objects - - // Statistics about allocation of low-level fixed-size structures. - // Protected by FixAlloc locks. - uint64 stacks_inuse; // this number is included in heap_inuse above - uint64 stacks_sys; // always 0 in mstats - uint64 mspan_inuse; // MSpan structures - uint64 mspan_sys; - uint64 mcache_inuse; // MCache structures - uint64 mcache_sys; - uint64 buckhash_sys; // profiling bucket hash table - uint64 gc_sys; - uint64 other_sys; - - // Statistics about garbage collector. - // Protected by mheap or stopping the world during GC. - uint64 next_gc; // next GC (in heap_alloc time) - uint64 last_gc; // last GC (in absolute time) - uint64 pause_total_ns; - uint64 pause_ns[256]; // circular buffer of recent GC pause lengths - uint64 pause_end[256]; // circular buffer of recent GC end times (nanoseconds since 1970) - uint32 numgc; - bool enablegc; - bool debuggc; - - // Statistics about allocation size classes. - - struct MStatsBySize { - uint32 size; - uint64 nmalloc; - uint64 nfree; - } by_size[NumSizeClasses]; - - uint64 tinyallocs; // number of tiny allocations that didn't cause actual allocation; not exported to Go directly -}; - - -#define mstats runtime·memstats -extern MStats mstats; -void runtime·updatememstats(GCStats *stats); -void runtime·ReadMemStats(MStats *stats); - -// Size classes. Computed and initialized by InitSizes. -// -// SizeToClass(0 <= n <= MaxSmallSize) returns the size class, -// 1 <= sizeclass < NumSizeClasses, for n. -// Size class 0 is reserved to mean "not small". -// -// class_to_size[i] = largest size in class i -// class_to_allocnpages[i] = number of pages to allocate when -// making new objects in class i - -int32 runtime·SizeToClass(int32); -uintptr runtime·roundupsize(uintptr); -extern int32 runtime·class_to_size[NumSizeClasses]; -extern int32 runtime·class_to_allocnpages[NumSizeClasses]; -extern int8 runtime·size_to_class8[1024/8 + 1]; -extern int8 runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1]; -extern void runtime·InitSizes(void); - -typedef struct MCacheList MCacheList; -struct MCacheList -{ - MLink *list; - uint32 nlist; -}; - -typedef struct StackFreeList StackFreeList; -struct StackFreeList -{ - MLink *list; // linked list of free stacks - uintptr size; // total size of stacks in list -}; - -typedef struct SudoG SudoG; - -// Per-thread (in Go, per-P) cache for small objects. -// No locking needed because it is per-thread (per-P). -struct MCache -{ - // The following members are accessed on every malloc, - // so they are grouped here for better caching. - int32 next_sample; // trigger heap sample after allocating this many bytes - intptr local_cachealloc; // bytes allocated (or freed) from cache since last lock of heap - // Allocator cache for tiny objects w/o pointers. - // See "Tiny allocator" comment in malloc.goc. - byte* tiny; - uintptr tinysize; - uintptr local_tinyallocs; // number of tiny allocs not counted in other stats - // The rest is not accessed on every malloc. - MSpan* alloc[NumSizeClasses]; // spans to allocate from - - StackFreeList stackcache[NumStackOrders]; - - SudoG* sudogcache; - - void* gcworkbuf; - - // Local allocator stats, flushed during GC. - uintptr local_nlookup; // number of pointer lookups - uintptr local_largefree; // bytes freed for large objects (>MaxSmallSize) - uintptr local_nlargefree; // number of frees for large objects (>MaxSmallSize) - uintptr local_nsmallfree[NumSizeClasses]; // number of frees for small objects (<=MaxSmallSize) -}; - -MSpan* runtime·MCache_Refill(MCache *c, int32 sizeclass); -void runtime·MCache_ReleaseAll(MCache *c); -void runtime·stackcache_clear(MCache *c); -void runtime·gcworkbuffree(void *b); - -enum -{ - KindSpecialFinalizer = 1, - KindSpecialProfile = 2, - // Note: The finalizer special must be first because if we're freeing - // an object, a finalizer special will cause the freeing operation - // to abort, and we want to keep the other special records around - // if that happens. -}; - -typedef struct Special Special; -struct Special -{ - Special* next; // linked list in span - uint16 offset; // span offset of object - byte kind; // kind of Special -}; - -// The described object has a finalizer set for it. -typedef struct SpecialFinalizer SpecialFinalizer; -struct SpecialFinalizer -{ - Special special; - FuncVal* fn; - uintptr nret; - Type* fint; - PtrType* ot; -}; - -// The described object is being heap profiled. -typedef struct Bucket Bucket; // from mprof.h -typedef struct SpecialProfile SpecialProfile; -struct SpecialProfile -{ - Special special; - Bucket* b; -}; - -// An MSpan is a run of pages. -enum -{ - MSpanInUse = 0, // allocated for garbage collected heap - MSpanStack, // allocated for use by stack allocator - MSpanFree, - MSpanListHead, - MSpanDead, -}; -struct MSpan -{ - MSpan *next; // in a span linked list - MSpan *prev; // in a span linked list - pageID start; // starting page number - uintptr npages; // number of pages in span - MLink *freelist; // list of free objects - // sweep generation: - // if sweepgen == h->sweepgen - 2, the span needs sweeping - // if sweepgen == h->sweepgen - 1, the span is currently being swept - // if sweepgen == h->sweepgen, the span is swept and ready to use - // h->sweepgen is incremented by 2 after every GC - uint32 sweepgen; - uint16 ref; // capacity - number of objects in freelist - uint8 sizeclass; // size class - bool incache; // being used by an MCache - uint8 state; // MSpanInUse etc - uint8 needzero; // needs to be zeroed before allocation - uintptr elemsize; // computed from sizeclass or from npages - int64 unusedsince; // First time spotted by GC in MSpanFree state - uintptr npreleased; // number of pages released to the OS - byte *limit; // end of data in span - Mutex specialLock; // guards specials list - Special *specials; // linked list of special records sorted by offset. -}; - -void runtime·MSpan_Init(MSpan *span, pageID start, uintptr npages); -void runtime·MSpan_EnsureSwept(MSpan *span); -bool runtime·MSpan_Sweep(MSpan *span, bool preserve); - -// Every MSpan is in one doubly-linked list, -// either one of the MHeap's free lists or one of the -// MCentral's span lists. We use empty MSpan structures as list heads. -void runtime·MSpanList_Init(MSpan *list); -bool runtime·MSpanList_IsEmpty(MSpan *list); -void runtime·MSpanList_Insert(MSpan *list, MSpan *span); -void runtime·MSpanList_InsertBack(MSpan *list, MSpan *span); -void runtime·MSpanList_Remove(MSpan *span); // from whatever list it is in - - -// Central list of free objects of a given size. -struct MCentral -{ - Mutex lock; - int32 sizeclass; - MSpan nonempty; // list of spans with a free object - MSpan empty; // list of spans with no free objects (or cached in an MCache) -}; - -void runtime·MCentral_Init(MCentral *c, int32 sizeclass); -MSpan* runtime·MCentral_CacheSpan(MCentral *c); -void runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s); -bool runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end, bool preserve); - -// Main malloc heap. -// The heap itself is the "free[]" and "large" arrays, -// but all the other global data is here too. -struct MHeap -{ - Mutex lock; - MSpan free[MaxMHeapList]; // free lists of given length - MSpan freelarge; // free lists length >= MaxMHeapList - MSpan busy[MaxMHeapList]; // busy lists of large objects of given length - MSpan busylarge; // busy lists of large objects length >= MaxMHeapList - MSpan **allspans; // all spans out there - MSpan **gcspans; // copy of allspans referenced by GC marker or sweeper - uint32 nspan; - uint32 nspancap; - uint32 sweepgen; // sweep generation, see comment in MSpan - uint32 sweepdone; // all spans are swept - - // span lookup - MSpan** spans; - uintptr spans_mapped; - - // range of addresses we might see in the heap - byte *bitmap; - uintptr bitmap_mapped; - byte *arena_start; - byte *arena_used; - byte *arena_end; - bool arena_reserved; - - // central free lists for small size classes. - // the padding makes sure that the MCentrals are - // spaced CacheLineSize bytes apart, so that each MCentral.lock - // gets its own cache line. - struct MHeapCentral { - MCentral mcentral; - byte pad[CacheLineSize]; - } central[NumSizeClasses]; - - FixAlloc spanalloc; // allocator for Span* - FixAlloc cachealloc; // allocator for MCache* - FixAlloc specialfinalizeralloc; // allocator for SpecialFinalizer* - FixAlloc specialprofilealloc; // allocator for SpecialProfile* - Mutex speciallock; // lock for sepcial record allocators. - - // Malloc stats. - uint64 largefree; // bytes freed for large objects (>MaxSmallSize) - uint64 nlargefree; // number of frees for large objects (>MaxSmallSize) - uint64 nsmallfree[NumSizeClasses]; // number of frees for small objects (<=MaxSmallSize) -}; -#define runtime·mheap runtime·mheap_ -extern MHeap runtime·mheap; - -void runtime·MHeap_Init(MHeap *h); -MSpan* runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero); -MSpan* runtime·MHeap_AllocStack(MHeap *h, uintptr npage); -void runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct); -void runtime·MHeap_FreeStack(MHeap *h, MSpan *s); -MSpan* runtime·MHeap_Lookup(MHeap *h, void *v); -MSpan* runtime·MHeap_LookupMaybe(MHeap *h, void *v); -void* runtime·MHeap_SysAlloc(MHeap *h, uintptr n); -void runtime·MHeap_MapBits(MHeap *h); -void runtime·MHeap_MapSpans(MHeap *h); -void runtime·MHeap_Scavenge(int32 k, uint64 now, uint64 limit); - -void* runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat); -int32 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **s); -uintptr runtime·sweepone(void); -void runtime·markspan(void *v, uintptr size, uintptr n, bool leftover); -void runtime·unmarkspan(void *v, uintptr size); -void runtime·purgecachedstats(MCache*); -void runtime·tracealloc(void*, uintptr, Type*); -void runtime·tracefree(void*, uintptr); -void runtime·tracegc(void); - -int32 runtime·gcpercent; -int32 runtime·readgogc(void); -void runtime·clearpools(void); - -enum -{ - // flags to malloc - FlagNoScan = 1<<0, // GC doesn't have to scan object - FlagNoZero = 1<<1, // don't zero memory -}; - -void runtime·mProf_Malloc(void*, uintptr); -void runtime·mProf_Free(Bucket*, uintptr, bool); -void runtime·mProf_GC(void); -void runtime·iterate_memprof(void (**callback)(Bucket*, uintptr, uintptr*, uintptr, uintptr, uintptr)); -int32 runtime·gcprocs(void); -void runtime·helpgc(int32 nproc); -void runtime·gchelper(void); -void runtime·createfing(void); -G* runtime·wakefing(void); -void runtime·getgcmask(byte*, Type*, byte**, uintptr*); - -// NOTE: Layout known to queuefinalizer. -typedef struct Finalizer Finalizer; -struct Finalizer -{ - FuncVal *fn; // function to call - void *arg; // ptr to object - uintptr nret; // bytes of return values from fn - Type *fint; // type of first argument of fn - PtrType *ot; // type of ptr to object -}; - -typedef struct FinBlock FinBlock; -struct FinBlock -{ - FinBlock *alllink; - FinBlock *next; - int32 cnt; - int32 cap; - Finalizer fin[1]; -}; -extern Mutex runtime·finlock; // protects the following variables -extern G* runtime·fing; -extern bool runtime·fingwait; -extern bool runtime·fingwake; -extern FinBlock *runtime·finq; // list of finalizers that are to be executed -extern FinBlock *runtime·finc; // cache of free blocks - -void runtime·setprofilebucket_m(void); - -bool runtime·addfinalizer(void*, FuncVal *fn, uintptr, Type*, PtrType*); -void runtime·removefinalizer(void*); -void runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot); -bool runtime·freespecial(Special *s, void *p, uintptr size, bool freed); - -// Information from the compiler about the layout of stack frames. -struct BitVector -{ - int32 n; // # of bits - uint8 *bytedata; -}; -typedef struct StackMap StackMap; -struct StackMap -{ - int32 n; // number of bitmaps - int32 nbit; // number of bits in each bitmap - uint8 bytedata[]; // bitmaps, each starting on a 32-bit boundary -}; -// Returns pointer map data for the given stackmap index -// (the index is encoded in PCDATA_StackMapIndex). -BitVector runtime·stackmapdata(StackMap *stackmap, int32 n); - -extern BitVector runtime·gcdatamask; -extern BitVector runtime·gcbssmask; - -// defined in mgc0.go -void runtime·gc_m_ptr(Eface*); -void runtime·gc_g_ptr(Eface*); -void runtime·gc_itab_ptr(Eface*); - -void runtime·setgcpercent_m(void); - -// Value we use to mark dead pointers when GODEBUG=gcdead=1. -#define PoisonGC ((uintptr)0xf969696969696969ULL) -#define PoisonStack ((uintptr)0x6868686868686868ULL) diff --git a/src/runtime/malloc1.go b/src/runtime/malloc1.go new file mode 100644 index 000000000..db02d9cca --- /dev/null +++ b/src/runtime/malloc1.go @@ -0,0 +1,318 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// See malloc.h for overview. +// +// TODO(rsc): double-check stats. + +package runtime + +import "unsafe" + +const _MaxArena32 = 2 << 30 + +// For use by Go. If it were a C enum it would be made available automatically, +// but the value of MaxMem is too large for enum. +// XXX - uintptr runtime·maxmem = MaxMem; + +func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { + _g_ := getg() + + _g_.m.mcache.local_nlookup++ + if ptrSize == 4 && _g_.m.mcache.local_nlookup >= 1<<30 { + // purge cache stats to prevent overflow + lock(&mheap_.lock) + purgecachedstats(_g_.m.mcache) + unlock(&mheap_.lock) + } + + s := mHeap_LookupMaybe(&mheap_, unsafe.Pointer(v)) + if sp != nil { + *sp = s + } + if s == nil { + if base != nil { + *base = 0 + } + if size != nil { + *size = 0 + } + return 0 + } + + p := uintptr(s.start) << _PageShift + if s.sizeclass == 0 { + // Large object. + if base != nil { + *base = p + } + if size != nil { + *size = s.npages << _PageShift + } + return 1 + } + + n := s.elemsize + if base != nil { + i := (uintptr(v) - uintptr(p)) / n + *base = p + i*n + } + if size != nil { + *size = n + } + + return 1 +} + +//go:nosplit +func purgecachedstats(c *mcache) { + // Protected by either heap or GC lock. + h := &mheap_ + memstats.heap_alloc += uint64(c.local_cachealloc) + c.local_cachealloc = 0 + memstats.tinyallocs += uint64(c.local_tinyallocs) + c.local_tinyallocs = 0 + memstats.nlookup += uint64(c.local_nlookup) + c.local_nlookup = 0 + h.largefree += uint64(c.local_largefree) + c.local_largefree = 0 + h.nlargefree += uint64(c.local_nlargefree) + c.local_nlargefree = 0 + for i := 0; i < len(c.local_nsmallfree); i++ { + h.nsmallfree[i] += uint64(c.local_nsmallfree[i]) + c.local_nsmallfree[i] = 0 + } +} + +func mallocinit() { + initSizes() + + if class_to_size[_TinySizeClass] != _TinySize { + gothrow("bad TinySizeClass") + } + + var p, arena_size, bitmap_size, spans_size, p_size, limit uintptr + var reserved bool + + // limit = runtime.memlimit(); + // See https://code.google.com/p/go/issues/detail?id=5049 + // TODO(rsc): Fix after 1.1. + limit = 0 + + // Set up the allocation arena, a contiguous area of memory where + // allocated data will be found. The arena begins with a bitmap large + // enough to hold 4 bits per allocated word. + if ptrSize == 8 && (limit == 0 || limit > 1<<30) { + // On a 64-bit machine, allocate from a single contiguous reservation. + // 128 GB (MaxMem) should be big enough for now. + // + // The code will work with the reservation at any address, but ask + // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). + // Allocating a 128 GB region takes away 37 bits, and the amd64 + // doesn't let us choose the top 17 bits, so that leaves the 11 bits + // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means + // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. + // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid + // UTF-8 sequences, and they are otherwise as far away from + // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 + // addresses. An earlier attempt to use 0x11f8 caused out of memory errors + // on OS X during thread allocations. 0x00c0 causes conflicts with + // AddressSanitizer which reserves all memory up to 0x0100. + // These choices are both for debuggability and to reduce the + // odds of the conservative garbage collector not collecting memory + // because some non-pointer block of memory had a bit pattern + // that matched a memory address. + // + // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) + // but it hardly matters: e0 00 is not valid UTF-8 either. + // + // If this fails we fall back to the 32 bit memory mechanism + arena_size = round(_MaxMem, _PageSize) + bitmap_size = arena_size / (ptrSize * 8 / 4) + spans_size = arena_size / _PageSize * ptrSize + spans_size = round(spans_size, _PageSize) + for i := 0; i <= 0x7f; i++ { + p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32) + p_size = bitmap_size + spans_size + arena_size + _PageSize + p = uintptr(sysReserve(unsafe.Pointer(p), p_size, &reserved)) + if p != 0 { + break + } + } + } + + if p == 0 { + // On a 32-bit machine, we can't typically get away + // with a giant virtual address space reservation. + // Instead we map the memory information bitmap + // immediately after the data segment, large enough + // to handle another 2GB of mappings (256 MB), + // along with a reservation for another 512 MB of memory. + // When that gets used up, we'll start asking the kernel + // for any memory anywhere and hope it's in the 2GB + // following the bitmap (presumably the executable begins + // near the bottom of memory, so we'll have to use up + // most of memory before the kernel resorts to giving out + // memory before the beginning of the text segment). + // + // Alternatively we could reserve 512 MB bitmap, enough + // for 4GB of mappings, and then accept any memory the + // kernel threw at us, but normally that's a waste of 512 MB + // of address space, which is probably too much in a 32-bit world. + bitmap_size = _MaxArena32 / (ptrSize * 8 / 4) + arena_size = 512 << 20 + spans_size = _MaxArena32 / _PageSize * ptrSize + if limit > 0 && arena_size+bitmap_size+spans_size > limit { + bitmap_size = (limit / 9) &^ ((1 << _PageShift) - 1) + arena_size = bitmap_size * 8 + spans_size = arena_size / _PageSize * ptrSize + } + spans_size = round(spans_size, _PageSize) + + // SysReserve treats the address we ask for, end, as a hint, + // not as an absolute requirement. If we ask for the end + // of the data segment but the operating system requires + // a little more space before we can start allocating, it will + // give out a slightly higher pointer. Except QEMU, which + // is buggy, as usual: it won't adjust the pointer upward. + // So adjust it upward a little bit ourselves: 1/4 MB to get + // away from the running binary image and then round up + // to a MB boundary. + p = round(uintptr(unsafe.Pointer(&end))+(1<<18), 1<<20) + p_size = bitmap_size + spans_size + arena_size + _PageSize + p = uintptr(sysReserve(unsafe.Pointer(p), p_size, &reserved)) + if p == 0 { + gothrow("runtime: cannot reserve arena virtual address space") + } + } + + // PageSize can be larger than OS definition of page size, + // so SysReserve can give us a PageSize-unaligned pointer. + // To overcome this we ask for PageSize more and round up the pointer. + p1 := round(p, _PageSize) + + mheap_.spans = (**mspan)(unsafe.Pointer(p1)) + mheap_.bitmap = p1 + spans_size + mheap_.arena_start = p1 + (spans_size + bitmap_size) + mheap_.arena_used = mheap_.arena_start + mheap_.arena_end = p + p_size + mheap_.arena_reserved = reserved + + if mheap_.arena_start&(_PageSize-1) != 0 { + println("bad pagesize", hex(p), hex(p1), hex(spans_size), hex(bitmap_size), hex(_PageSize), "start", hex(mheap_.arena_start)) + gothrow("misrounded allocation in mallocinit") + } + + // Initialize the rest of the allocator. + mHeap_Init(&mheap_, spans_size) + _g_ := getg() + _g_.m.mcache = allocmcache() +} + +func mHeap_SysAlloc(h *mheap, n uintptr) unsafe.Pointer { + if n > uintptr(h.arena_end)-uintptr(h.arena_used) { + // We are in 32-bit mode, maybe we didn't use all possible address space yet. + // Reserve some more space. + p_size := round(n+_PageSize, 256<<20) + new_end := h.arena_end + p_size + if new_end <= h.arena_start+_MaxArena32 { + // TODO: It would be bad if part of the arena + // is reserved and part is not. + var reserved bool + p := uintptr(sysReserve((unsafe.Pointer)(h.arena_end), p_size, &reserved)) + if p == h.arena_end { + h.arena_end = new_end + h.arena_reserved = reserved + } else if p+p_size <= h.arena_start+_MaxArena32 { + // Keep everything page-aligned. + // Our pages are bigger than hardware pages. + h.arena_end = p + p_size + h.arena_used = p + (-uintptr(p) & (_PageSize - 1)) + h.arena_reserved = reserved + } else { + var stat uint64 + sysFree((unsafe.Pointer)(p), p_size, &stat) + } + } + } + + if n <= uintptr(h.arena_end)-uintptr(h.arena_used) { + // Keep taking from our reservation. + p := h.arena_used + sysMap((unsafe.Pointer)(p), n, h.arena_reserved, &memstats.heap_sys) + h.arena_used += n + mHeap_MapBits(h) + mHeap_MapSpans(h) + if raceenabled { + racemapshadow((unsafe.Pointer)(p), n) + } + + if uintptr(p)&(_PageSize-1) != 0 { + gothrow("misrounded allocation in MHeap_SysAlloc") + } + return (unsafe.Pointer)(p) + } + + // If using 64-bit, our reservation is all we have. + if uintptr(h.arena_end)-uintptr(h.arena_start) >= _MaxArena32 { + return nil + } + + // On 32-bit, once the reservation is gone we can + // try to get memory at a location chosen by the OS + // and hope that it is in the range we allocated bitmap for. + p_size := round(n, _PageSize) + _PageSize + p := uintptr(sysAlloc(p_size, &memstats.heap_sys)) + if p == 0 { + return nil + } + + if p < h.arena_start || uintptr(p)+p_size-uintptr(h.arena_start) >= _MaxArena32 { + print("runtime: memory allocated by OS (", p, ") not in usable range [", hex(h.arena_start), ",", hex(h.arena_start+_MaxArena32), ")\n") + sysFree((unsafe.Pointer)(p), p_size, &memstats.heap_sys) + return nil + } + + p_end := p + p_size + p += -p & (_PageSize - 1) + if uintptr(p)+n > uintptr(h.arena_used) { + h.arena_used = p + n + if p_end > h.arena_end { + h.arena_end = p_end + } + mHeap_MapBits(h) + mHeap_MapSpans(h) + if raceenabled { + racemapshadow((unsafe.Pointer)(p), n) + } + } + + if uintptr(p)&(_PageSize-1) != 0 { + gothrow("misrounded allocation in MHeap_SysAlloc") + } + return (unsafe.Pointer)(p) +} + +var end struct{} + +func largeAlloc(size uintptr, flag uint32) *mspan { + // print("largeAlloc size=", size, "\n") + + if size+_PageSize < size { + gothrow("out of memory") + } + npages := size >> _PageShift + if size&_PageMask != 0 { + npages++ + } + s := mHeap_Alloc(&mheap_, npages, 0, true, flag&_FlagNoZero == 0) + if s == nil { + gothrow("out of memory") + } + s.limit = uintptr(s.start)<<_PageShift + size + v := unsafe.Pointer(uintptr(s.start) << _PageShift) + // setup for mark sweep + markspan(v, 0, 0, true) + return s +} diff --git a/src/runtime/malloc2.go b/src/runtime/malloc2.go new file mode 100644 index 000000000..c175c2aec --- /dev/null +++ b/src/runtime/malloc2.go @@ -0,0 +1,475 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Memory allocator, based on tcmalloc. +// http://goog-perftools.sourceforge.net/doc/tcmalloc.html + +// The main allocator works in runs of pages. +// Small allocation sizes (up to and including 32 kB) are +// rounded to one of about 100 size classes, each of which +// has its own free list of objects of exactly that size. +// Any free page of memory can be split into a set of objects +// of one size class, which are then managed using free list +// allocators. +// +// The allocator's data structures are: +// +// FixAlloc: a free-list allocator for fixed-size objects, +// used to manage storage used by the allocator. +// MHeap: the malloc heap, managed at page (4096-byte) granularity. +// MSpan: a run of pages managed by the MHeap. +// MCentral: a shared free list for a given size class. +// MCache: a per-thread (in Go, per-P) cache for small objects. +// MStats: allocation statistics. +// +// Allocating a small object proceeds up a hierarchy of caches: +// +// 1. Round the size up to one of the small size classes +// and look in the corresponding MCache free list. +// If the list is not empty, allocate an object from it. +// This can all be done without acquiring a lock. +// +// 2. If the MCache free list is empty, replenish it by +// taking a bunch of objects from the MCentral free list. +// Moving a bunch amortizes the cost of acquiring the MCentral lock. +// +// 3. If the MCentral free list is empty, replenish it by +// allocating a run of pages from the MHeap and then +// chopping that memory into a objects of the given size. +// Allocating many objects amortizes the cost of locking +// the heap. +// +// 4. If the MHeap is empty or has no page runs large enough, +// allocate a new group of pages (at least 1MB) from the +// operating system. Allocating a large run of pages +// amortizes the cost of talking to the operating system. +// +// Freeing a small object proceeds up the same hierarchy: +// +// 1. Look up the size class for the object and add it to +// the MCache free list. +// +// 2. If the MCache free list is too long or the MCache has +// too much memory, return some to the MCentral free lists. +// +// 3. If all the objects in a given span have returned to +// the MCentral list, return that span to the page heap. +// +// 4. If the heap has too much memory, return some to the +// operating system. +// +// TODO(rsc): Step 4 is not implemented. +// +// Allocating and freeing a large object uses the page heap +// directly, bypassing the MCache and MCentral free lists. +// +// The small objects on the MCache and MCentral free lists +// may or may not be zeroed. They are zeroed if and only if +// the second word of the object is zero. A span in the +// page heap is zeroed unless s->needzero is set. When a span +// is allocated to break into small objects, it is zeroed if needed +// and s->needzero is set. There are two main benefits to delaying the +// zeroing this way: +// +// 1. stack frames allocated from the small object lists +// or the page heap can avoid zeroing altogether. +// 2. the cost of zeroing when reusing a small object is +// charged to the mutator, not the garbage collector. +// +// This C code was written with an eye toward translating to Go +// in the future. Methods have the form Type_Method(Type *t, ...). + +const ( + _PageShift = 13 + _PageSize = 1 << _PageShift + _PageMask = _PageSize - 1 +) + +const ( + // _64bit = 1 on 64-bit systems, 0 on 32-bit systems + _64bit = 1 << (^uintptr(0) >> 63) / 2 + + // Computed constant. The definition of MaxSmallSize and the + // algorithm in msize.c produce some number of different allocation + // size classes. NumSizeClasses is that number. It's needed here + // because there are static arrays of this length; when msize runs its + // size choosing algorithm it double-checks that NumSizeClasses agrees. + _NumSizeClasses = 67 + + // Tunable constants. + _MaxSmallSize = 32 << 10 + + // Tiny allocator parameters, see "Tiny allocator" comment in malloc.goc. + _TinySize = 16 + _TinySizeClass = 2 + + _FixAllocChunk = 16 << 10 // Chunk size for FixAlloc + _MaxMHeapList = 1 << (20 - _PageShift) // Maximum page length for fixed-size list in MHeap. + _HeapAllocChunk = 1 << 20 // Chunk size for heap growth + + // Per-P, per order stack segment cache size. + _StackCacheSize = 32 * 1024 + + // Number of orders that get caching. Order 0 is FixedStack + // and each successive order is twice as large. + _NumStackOrders = 3 + + // Number of bits in page to span calculations (4k pages). + // On Windows 64-bit we limit the arena to 32GB or 35 bits. + // Windows counts memory used by page table into committed memory + // of the process, so we can't reserve too much memory. + // See http://golang.org/issue/5402 and http://golang.org/issue/5236. + // On other 64-bit platforms, we limit the arena to 128GB, or 37 bits. + // On 32-bit, we don't bother limiting anything, so we use the full 32-bit address. + _MHeapMap_TotalBits = (_64bit*goos_windows)*35 + (_64bit*(1-goos_windows))*37 + (1-_64bit)*32 + _MHeapMap_Bits = _MHeapMap_TotalBits - _PageShift + + _MaxMem = uintptr(1<<_MHeapMap_TotalBits - 1) + + // Max number of threads to run garbage collection. + // 2, 3, and 4 are all plausible maximums depending + // on the hardware details of the machine. The garbage + // collector scales well to 32 cpus. + _MaxGcproc = 32 +) + +// A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) +type mlink struct { + next *mlink +} + +// sysAlloc obtains a large chunk of zeroed memory from the +// operating system, typically on the order of a hundred kilobytes +// or a megabyte. +// NOTE: sysAlloc returns OS-aligned memory, but the heap allocator +// may use larger alignment, so the caller must be careful to realign the +// memory obtained by sysAlloc. +// +// SysUnused notifies the operating system that the contents +// of the memory region are no longer needed and can be reused +// for other purposes. +// SysUsed notifies the operating system that the contents +// of the memory region are needed again. +// +// SysFree returns it unconditionally; this is only used if +// an out-of-memory error has been detected midway through +// an allocation. It is okay if SysFree is a no-op. +// +// SysReserve reserves address space without allocating memory. +// If the pointer passed to it is non-nil, the caller wants the +// reservation there, but SysReserve can still choose another +// location if that one is unavailable. On some systems and in some +// cases SysReserve will simply check that the address space is +// available and not actually reserve it. If SysReserve returns +// non-nil, it sets *reserved to true if the address space is +// reserved, false if it has merely been checked. +// NOTE: SysReserve returns OS-aligned memory, but the heap allocator +// may use larger alignment, so the caller must be careful to realign the +// memory obtained by sysAlloc. +// +// SysMap maps previously reserved address space for use. +// The reserved argument is true if the address space was really +// reserved, not merely checked. +// +// SysFault marks a (already sysAlloc'd) region to fault +// if accessed. Used only for debugging the runtime. + +// FixAlloc is a simple free-list allocator for fixed size objects. +// Malloc uses a FixAlloc wrapped around sysAlloc to manages its +// MCache and MSpan objects. +// +// Memory returned by FixAlloc_Alloc is not zeroed. +// The caller is responsible for locking around FixAlloc calls. +// Callers can keep state in the object but the first word is +// smashed by freeing and reallocating. +type fixalloc struct { + size uintptr + first unsafe.Pointer // go func(unsafe.pointer, unsafe.pointer); f(arg, p) called first time p is returned + arg unsafe.Pointer + list *mlink + chunk *byte + nchunk uint32 + inuse uintptr // in-use bytes now + stat *uint64 +} + +// Statistics. +// Shared with Go: if you edit this structure, also edit type MemStats in mem.go. +type mstats struct { + // General statistics. + alloc uint64 // bytes allocated and still in use + total_alloc uint64 // bytes allocated (even if freed) + sys uint64 // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) + nlookup uint64 // number of pointer lookups + nmalloc uint64 // number of mallocs + nfree uint64 // number of frees + + // Statistics about malloc heap. + // protected by mheap.lock + heap_alloc uint64 // bytes allocated and still in use + heap_sys uint64 // bytes obtained from system + heap_idle uint64 // bytes in idle spans + heap_inuse uint64 // bytes in non-idle spans + heap_released uint64 // bytes released to the os + heap_objects uint64 // total number of allocated objects + + // Statistics about allocation of low-level fixed-size structures. + // Protected by FixAlloc locks. + stacks_inuse uint64 // this number is included in heap_inuse above + stacks_sys uint64 // always 0 in mstats + mspan_inuse uint64 // mspan structures + mspan_sys uint64 + mcache_inuse uint64 // mcache structures + mcache_sys uint64 + buckhash_sys uint64 // profiling bucket hash table + gc_sys uint64 + other_sys uint64 + + // Statistics about garbage collector. + // Protected by mheap or stopping the world during GC. + next_gc uint64 // next gc (in heap_alloc time) + last_gc uint64 // last gc (in absolute time) + pause_total_ns uint64 + pause_ns [256]uint64 // circular buffer of recent gc pause lengths + pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) + numgc uint32 + enablegc bool + debuggc bool + + // Statistics about allocation size classes. + + by_size [_NumSizeClasses]struct { + size uint32 + nmalloc uint64 + nfree uint64 + } + + tinyallocs uint64 // number of tiny allocations that didn't cause actual allocation; not exported to go directly +} + +var memstats mstats + +// Size classes. Computed and initialized by InitSizes. +// +// SizeToClass(0 <= n <= MaxSmallSize) returns the size class, +// 1 <= sizeclass < NumSizeClasses, for n. +// Size class 0 is reserved to mean "not small". +// +// class_to_size[i] = largest size in class i +// class_to_allocnpages[i] = number of pages to allocate when +// making new objects in class i + +var class_to_size [_NumSizeClasses]int32 +var class_to_allocnpages [_NumSizeClasses]int32 +var size_to_class8 [1024/8 + 1]int8 +var size_to_class128 [(_MaxSmallSize-1024)/128 + 1]int8 + +type mcachelist struct { + list *mlink + nlist uint32 +} + +type stackfreelist struct { + list *mlink // linked list of free stacks + size uintptr // total size of stacks in list +} + +// Per-thread (in Go, per-P) cache for small objects. +// No locking needed because it is per-thread (per-P). +type mcache struct { + // The following members are accessed on every malloc, + // so they are grouped here for better caching. + next_sample int32 // trigger heap sample after allocating this many bytes + local_cachealloc intptr // bytes allocated (or freed) from cache since last lock of heap + // Allocator cache for tiny objects w/o pointers. + // See "Tiny allocator" comment in malloc.goc. + tiny *byte + tinysize uintptr + local_tinyallocs uintptr // number of tiny allocs not counted in other stats + + // The rest is not accessed on every malloc. + alloc [_NumSizeClasses]*mspan // spans to allocate from + + stackcache [_NumStackOrders]stackfreelist + + sudogcache *sudog + + gcworkbuf unsafe.Pointer + + // Local allocator stats, flushed during GC. + local_nlookup uintptr // number of pointer lookups + local_largefree uintptr // bytes freed for large objects (>maxsmallsize) + local_nlargefree uintptr // number of frees for large objects (>maxsmallsize) + local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize) +} + +const ( + _KindSpecialFinalizer = 1 + _KindSpecialProfile = 2 + // Note: The finalizer special must be first because if we're freeing + // an object, a finalizer special will cause the freeing operation + // to abort, and we want to keep the other special records around + // if that happens. +) + +type special struct { + next *special // linked list in span + offset uint16 // span offset of object + kind byte // kind of special +} + +// The described object has a finalizer set for it. +type specialfinalizer struct { + special special + fn *funcval + nret uintptr + fint *_type + ot *ptrtype +} + +// The described object is being heap profiled. +type specialprofile struct { + special special + b *bucket +} + +// An MSpan is a run of pages. +const ( + _MSpanInUse = iota // allocated for garbage collected heap + _MSpanStack // allocated for use by stack allocator + _MSpanFree + _MSpanListHead + _MSpanDead +) + +type mspan struct { + next *mspan // in a span linked list + prev *mspan // in a span linked list + start pageID // starting page number + npages uintptr // number of pages in span + freelist *mlink // list of free objects + // sweep generation: + // if sweepgen == h->sweepgen - 2, the span needs sweeping + // if sweepgen == h->sweepgen - 1, the span is currently being swept + // if sweepgen == h->sweepgen, the span is swept and ready to use + // h->sweepgen is incremented by 2 after every GC + sweepgen uint32 + ref uint16 // capacity - number of objects in freelist + sizeclass uint8 // size class + incache bool // being used by an mcache + state uint8 // mspaninuse etc + needzero uint8 // needs to be zeroed before allocation + elemsize uintptr // computed from sizeclass or from npages + unusedsince int64 // first time spotted by gc in mspanfree state + npreleased uintptr // number of pages released to the os + limit uintptr // end of data in span + speciallock mutex // guards specials list + specials *special // linked list of special records sorted by offset. +} + +// Every MSpan is in one doubly-linked list, +// either one of the MHeap's free lists or one of the +// MCentral's span lists. We use empty MSpan structures as list heads. + +// Central list of free objects of a given size. +type mcentral struct { + lock mutex + sizeclass int32 + nonempty mspan // list of spans with a free object + empty mspan // list of spans with no free objects (or cached in an mcache) +} + +// Main malloc heap. +// The heap itself is the "free[]" and "large" arrays, +// but all the other global data is here too. +type mheap struct { + lock mutex + free [_MaxMHeapList]mspan // free lists of given length + freelarge mspan // free lists length >= _MaxMHeapList + busy [_MaxMHeapList]mspan // busy lists of large objects of given length + busylarge mspan // busy lists of large objects length >= _MaxMHeapList + allspans **mspan // all spans out there + gcspans **mspan // copy of allspans referenced by gc marker or sweeper + nspan uint32 + sweepgen uint32 // sweep generation, see comment in mspan + sweepdone uint32 // all spans are swept + + // span lookup + spans **mspan + spans_mapped uintptr + + // range of addresses we might see in the heap + bitmap uintptr + bitmap_mapped uintptr + arena_start uintptr + arena_used uintptr + arena_end uintptr + arena_reserved bool + + // central free lists for small size classes. + // the padding makes sure that the MCentrals are + // spaced CacheLineSize bytes apart, so that each MCentral.lock + // gets its own cache line. + central [_NumSizeClasses]struct { + mcentral mcentral + pad [_CacheLineSize]byte + } + + spanalloc fixalloc // allocator for span* + cachealloc fixalloc // allocator for mcache* + specialfinalizeralloc fixalloc // allocator for specialfinalizer* + specialprofilealloc fixalloc // allocator for specialprofile* + speciallock mutex // lock for sepcial record allocators. + + // Malloc stats. + largefree uint64 // bytes freed for large objects (>maxsmallsize) + nlargefree uint64 // number of frees for large objects (>maxsmallsize) + nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize) +} + +var mheap_ mheap + +const ( + // flags to malloc + _FlagNoScan = 1 << 0 // GC doesn't have to scan object + _FlagNoZero = 1 << 1 // don't zero memory +) + +// NOTE: Layout known to queuefinalizer. +type finalizer struct { + fn *funcval // function to call + arg unsafe.Pointer // ptr to object + nret uintptr // bytes of return values from fn + fint *_type // type of first argument of fn + ot *ptrtype // type of ptr to object +} + +type finblock struct { + alllink *finblock + next *finblock + cnt int32 + cap int32 + fin [1]finalizer +} + +// Information from the compiler about the layout of stack frames. +type bitvector struct { + n int32 // # of bits + bytedata *uint8 +} + +type stackmap struct { + n int32 // number of bitmaps + nbit int32 // number of bits in each bitmap + bytedata [0]byte // bitmaps, each starting on a 32-bit boundary +} + +// Returns pointer map data for the given stackmap index +// (the index is encoded in PCDATA_StackMapIndex). + +// defined in mgc0.go diff --git a/src/runtime/mcache.c b/src/runtime/mcache.c deleted file mode 100644 index 5fdbe3266..000000000 --- a/src/runtime/mcache.c +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Per-P malloc cache for small objects. -// -// See malloc.h for an overview. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -extern volatile intgo runtime·MemProfileRate; - -// dummy MSpan that contains no free objects. -MSpan runtime·emptymspan; - -MCache* -runtime·allocmcache(void) -{ - intgo rate; - MCache *c; - int32 i; - - runtime·lock(&runtime·mheap.lock); - c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc); - runtime·unlock(&runtime·mheap.lock); - runtime·memclr((byte*)c, sizeof(*c)); - for(i = 0; i < NumSizeClasses; i++) - c->alloc[i] = &runtime·emptymspan; - - // Set first allocation sample size. - rate = runtime·MemProfileRate; - if(rate > 0x3fffffff) // make 2*rate not overflow - rate = 0x3fffffff; - if(rate != 0) - c->next_sample = runtime·fastrand1() % (2*rate); - - return c; -} - -static void -freemcache(MCache *c) -{ - runtime·MCache_ReleaseAll(c); - runtime·stackcache_clear(c); - runtime·gcworkbuffree(c->gcworkbuf); - runtime·lock(&runtime·mheap.lock); - runtime·purgecachedstats(c); - runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c); - runtime·unlock(&runtime·mheap.lock); -} - -static void -freemcache_m(void) -{ - MCache *c; - - c = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - freemcache(c); -} - -void -runtime·freemcache(MCache *c) -{ - void (*fn)(void); - - g->m->ptrarg[0] = c; - fn = freemcache_m; - runtime·onM(&fn); -} - -// Gets a span that has a free object in it and assigns it -// to be the cached span for the given sizeclass. Returns this span. -MSpan* -runtime·MCache_Refill(MCache *c, int32 sizeclass) -{ - MSpan *s; - - g->m->locks++; - // Return the current cached span to the central lists. - s = c->alloc[sizeclass]; - if(s->freelist != nil) - runtime·throw("refill on a nonempty span"); - if(s != &runtime·emptymspan) - s->incache = false; - - // Get a new cached span from the central lists. - s = runtime·MCentral_CacheSpan(&runtime·mheap.central[sizeclass].mcentral); - if(s == nil) - runtime·throw("out of memory"); - if(s->freelist == nil) { - runtime·printf("%d %d\n", s->ref, (int32)((s->npages << PageShift) / s->elemsize)); - runtime·throw("empty span"); - } - c->alloc[sizeclass] = s; - g->m->locks--; - return s; -} - -void -runtime·MCache_ReleaseAll(MCache *c) -{ - int32 i; - MSpan *s; - - for(i=0; i<NumSizeClasses; i++) { - s = c->alloc[i]; - if(s != &runtime·emptymspan) { - runtime·MCentral_UncacheSpan(&runtime·mheap.central[i].mcentral, s); - c->alloc[i] = &runtime·emptymspan; - } - } -} diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go new file mode 100644 index 000000000..d3afef6be --- /dev/null +++ b/src/runtime/mcache.go @@ -0,0 +1,86 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Per-P malloc cache for small objects. +// +// See malloc.h for an overview. + +package runtime + +import "unsafe" + +// dummy MSpan that contains no free objects. +var emptymspan mspan + +func allocmcache() *mcache { + lock(&mheap_.lock) + c := (*mcache)(fixAlloc_Alloc(&mheap_.cachealloc)) + unlock(&mheap_.lock) + memclr(unsafe.Pointer(c), unsafe.Sizeof(*c)) + for i := 0; i < _NumSizeClasses; i++ { + c.alloc[i] = &emptymspan + } + + // Set first allocation sample size. + rate := MemProfileRate + if rate > 0x3fffffff { // make 2*rate not overflow + rate = 0x3fffffff + } + if rate != 0 { + c.next_sample = int32(int(fastrand1()) % (2 * rate)) + } + + return c +} + +func freemcache(c *mcache) { + systemstack(func() { + mCache_ReleaseAll(c) + stackcache_clear(c) + gcworkbuffree(c.gcworkbuf) + lock(&mheap_.lock) + purgecachedstats(c) + fixAlloc_Free(&mheap_.cachealloc, unsafe.Pointer(c)) + unlock(&mheap_.lock) + }) +} + +// Gets a span that has a free object in it and assigns it +// to be the cached span for the given sizeclass. Returns this span. +func mCache_Refill(c *mcache, sizeclass int32) *mspan { + _g_ := getg() + + _g_.m.locks++ + // Return the current cached span to the central lists. + s := c.alloc[sizeclass] + if s.freelist != nil { + gothrow("refill on a nonempty span") + } + if s != &emptymspan { + s.incache = false + } + + // Get a new cached span from the central lists. + s = mCentral_CacheSpan(&mheap_.central[sizeclass].mcentral) + if s == nil { + gothrow("out of memory") + } + if s.freelist == nil { + println(s.ref, (s.npages<<_PageShift)/s.elemsize) + gothrow("empty span") + } + c.alloc[sizeclass] = s + _g_.m.locks-- + return s +} + +func mCache_ReleaseAll(c *mcache) { + for i := 0; i < _NumSizeClasses; i++ { + s := c.alloc[i] + if s != &emptymspan { + mCentral_UncacheSpan(&mheap_.central[i].mcentral, s) + c.alloc[i] = &emptymspan + } + } +} diff --git a/src/runtime/mcentral.c b/src/runtime/mcentral.c deleted file mode 100644 index fe6bcfeb1..000000000 --- a/src/runtime/mcentral.c +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Central free lists. -// -// See malloc.h for an overview. -// -// The MCentral doesn't actually contain the list of free objects; the MSpan does. -// Each MCentral is two lists of MSpans: those with free objects (c->nonempty) -// and those that are completely allocated (c->empty). - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -static MSpan* MCentral_Grow(MCentral *c); - -// Initialize a single central free list. -void -runtime·MCentral_Init(MCentral *c, int32 sizeclass) -{ - c->sizeclass = sizeclass; - runtime·MSpanList_Init(&c->nonempty); - runtime·MSpanList_Init(&c->empty); -} - -// Allocate a span to use in an MCache. -MSpan* -runtime·MCentral_CacheSpan(MCentral *c) -{ - MSpan *s; - int32 cap, n; - uint32 sg; - - runtime·lock(&c->lock); - sg = runtime·mheap.sweepgen; -retry: - for(s = c->nonempty.next; s != &c->nonempty; s = s->next) { - if(s->sweepgen == sg-2 && runtime·cas(&s->sweepgen, sg-2, sg-1)) { - runtime·MSpanList_Remove(s); - runtime·MSpanList_InsertBack(&c->empty, s); - runtime·unlock(&c->lock); - runtime·MSpan_Sweep(s, true); - goto havespan; - } - if(s->sweepgen == sg-1) { - // the span is being swept by background sweeper, skip - continue; - } - // we have a nonempty span that does not require sweeping, allocate from it - runtime·MSpanList_Remove(s); - runtime·MSpanList_InsertBack(&c->empty, s); - runtime·unlock(&c->lock); - goto havespan; - } - - for(s = c->empty.next; s != &c->empty; s = s->next) { - if(s->sweepgen == sg-2 && runtime·cas(&s->sweepgen, sg-2, sg-1)) { - // we have an empty span that requires sweeping, - // sweep it and see if we can free some space in it - runtime·MSpanList_Remove(s); - // swept spans are at the end of the list - runtime·MSpanList_InsertBack(&c->empty, s); - runtime·unlock(&c->lock); - runtime·MSpan_Sweep(s, true); - if(s->freelist != nil) - goto havespan; - runtime·lock(&c->lock); - // the span is still empty after sweep - // it is already in the empty list, so just retry - goto retry; - } - if(s->sweepgen == sg-1) { - // the span is being swept by background sweeper, skip - continue; - } - // already swept empty span, - // all subsequent ones must also be either swept or in process of sweeping - break; - } - runtime·unlock(&c->lock); - - // Replenish central list if empty. - s = MCentral_Grow(c); - if(s == nil) - return nil; - runtime·lock(&c->lock); - runtime·MSpanList_InsertBack(&c->empty, s); - runtime·unlock(&c->lock); - -havespan: - // At this point s is a non-empty span, queued at the end of the empty list, - // c is unlocked. - cap = (s->npages << PageShift) / s->elemsize; - n = cap - s->ref; - if(n == 0) - runtime·throw("empty span"); - if(s->freelist == nil) - runtime·throw("freelist empty"); - s->incache = true; - return s; -} - -// Return span from an MCache. -void -runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s) -{ - int32 cap, n; - - runtime·lock(&c->lock); - - s->incache = false; - - if(s->ref == 0) - runtime·throw("uncaching full span"); - - cap = (s->npages << PageShift) / s->elemsize; - n = cap - s->ref; - if(n > 0) { - runtime·MSpanList_Remove(s); - runtime·MSpanList_Insert(&c->nonempty, s); - } - runtime·unlock(&c->lock); -} - -// Free n objects from a span s back into the central free list c. -// Called during sweep. -// Returns true if the span was returned to heap. Sets sweepgen to -// the latest generation. -// If preserve=true, don't return the span to heap nor relink in MCentral lists; -// caller takes care of it. -bool -runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end, bool preserve) -{ - bool wasempty; - - if(s->incache) - runtime·throw("freespan into cached span"); - - // Add the objects back to s's free list. - wasempty = s->freelist == nil; - end->next = s->freelist; - s->freelist = start; - s->ref -= n; - - if(preserve) { - // preserve is set only when called from MCentral_CacheSpan above, - // the span must be in the empty list. - if(s->next == nil) - runtime·throw("can't preserve unlinked span"); - runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen); - return false; - } - - runtime·lock(&c->lock); - - // Move to nonempty if necessary. - if(wasempty) { - runtime·MSpanList_Remove(s); - runtime·MSpanList_Insert(&c->nonempty, s); - } - - // delay updating sweepgen until here. This is the signal that - // the span may be used in an MCache, so it must come after the - // linked list operations above (actually, just after the - // lock of c above.) - runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen); - - if(s->ref != 0) { - runtime·unlock(&c->lock); - return false; - } - - // s is completely freed, return it to the heap. - runtime·MSpanList_Remove(s); - s->needzero = 1; - s->freelist = nil; - runtime·unlock(&c->lock); - runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); - runtime·MHeap_Free(&runtime·mheap, s, 0); - return true; -} - -// Fetch a new span from the heap and carve into objects for the free list. -static MSpan* -MCentral_Grow(MCentral *c) -{ - uintptr size, npages, i, n; - MLink **tailp, *v; - byte *p; - MSpan *s; - - npages = runtime·class_to_allocnpages[c->sizeclass]; - size = runtime·class_to_size[c->sizeclass]; - n = (npages << PageShift) / size; - s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1); - if(s == nil) - return nil; - - // Carve span into sequence of blocks. - tailp = &s->freelist; - p = (byte*)(s->start << PageShift); - s->limit = p + size*n; - for(i=0; i<n; i++) { - v = (MLink*)p; - *tailp = v; - tailp = &v->next; - p += size; - } - *tailp = nil; - runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); - return s; -} diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go new file mode 100644 index 000000000..0d172a08b --- /dev/null +++ b/src/runtime/mcentral.go @@ -0,0 +1,199 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Central free lists. +// +// See malloc.h for an overview. +// +// The MCentral doesn't actually contain the list of free objects; the MSpan does. +// Each MCentral is two lists of MSpans: those with free objects (c->nonempty) +// and those that are completely allocated (c->empty). + +package runtime + +import "unsafe" + +// Initialize a single central free list. +func mCentral_Init(c *mcentral, sizeclass int32) { + c.sizeclass = sizeclass + mSpanList_Init(&c.nonempty) + mSpanList_Init(&c.empty) +} + +// Allocate a span to use in an MCache. +func mCentral_CacheSpan(c *mcentral) *mspan { + lock(&c.lock) + sg := mheap_.sweepgen +retry: + var s *mspan + for s = c.nonempty.next; s != &c.nonempty; s = s.next { + if s.sweepgen == sg-2 && cas(&s.sweepgen, sg-2, sg-1) { + mSpanList_Remove(s) + mSpanList_InsertBack(&c.empty, s) + unlock(&c.lock) + mSpan_Sweep(s, true) + goto havespan + } + if s.sweepgen == sg-1 { + // the span is being swept by background sweeper, skip + continue + } + // we have a nonempty span that does not require sweeping, allocate from it + mSpanList_Remove(s) + mSpanList_InsertBack(&c.empty, s) + unlock(&c.lock) + goto havespan + } + + for s = c.empty.next; s != &c.empty; s = s.next { + if s.sweepgen == sg-2 && cas(&s.sweepgen, sg-2, sg-1) { + // we have an empty span that requires sweeping, + // sweep it and see if we can free some space in it + mSpanList_Remove(s) + // swept spans are at the end of the list + mSpanList_InsertBack(&c.empty, s) + unlock(&c.lock) + mSpan_Sweep(s, true) + if s.freelist != nil { + goto havespan + } + lock(&c.lock) + // the span is still empty after sweep + // it is already in the empty list, so just retry + goto retry + } + if s.sweepgen == sg-1 { + // the span is being swept by background sweeper, skip + continue + } + // already swept empty span, + // all subsequent ones must also be either swept or in process of sweeping + break + } + unlock(&c.lock) + + // Replenish central list if empty. + s = mCentral_Grow(c) + if s == nil { + return nil + } + lock(&c.lock) + mSpanList_InsertBack(&c.empty, s) + unlock(&c.lock) + + // At this point s is a non-empty span, queued at the end of the empty list, + // c is unlocked. +havespan: + cap := int32((s.npages << _PageShift) / s.elemsize) + n := cap - int32(s.ref) + if n == 0 { + gothrow("empty span") + } + if s.freelist == nil { + gothrow("freelist empty") + } + s.incache = true + return s +} + +// Return span from an MCache. +func mCentral_UncacheSpan(c *mcentral, s *mspan) { + lock(&c.lock) + + s.incache = false + + if s.ref == 0 { + gothrow("uncaching full span") + } + + cap := int32((s.npages << _PageShift) / s.elemsize) + n := cap - int32(s.ref) + if n > 0 { + mSpanList_Remove(s) + mSpanList_Insert(&c.nonempty, s) + } + unlock(&c.lock) +} + +// Free n objects from a span s back into the central free list c. +// Called during sweep. +// Returns true if the span was returned to heap. Sets sweepgen to +// the latest generation. +// If preserve=true, don't return the span to heap nor relink in MCentral lists; +// caller takes care of it. +func mCentral_FreeSpan(c *mcentral, s *mspan, n int32, start *mlink, end *mlink, preserve bool) bool { + if s.incache { + gothrow("freespan into cached span") + } + + // Add the objects back to s's free list. + wasempty := s.freelist == nil + end.next = s.freelist + s.freelist = start + s.ref -= uint16(n) + + if preserve { + // preserve is set only when called from MCentral_CacheSpan above, + // the span must be in the empty list. + if s.next == nil { + gothrow("can't preserve unlinked span") + } + atomicstore(&s.sweepgen, mheap_.sweepgen) + return false + } + + lock(&c.lock) + + // Move to nonempty if necessary. + if wasempty { + mSpanList_Remove(s) + mSpanList_Insert(&c.nonempty, s) + } + + // delay updating sweepgen until here. This is the signal that + // the span may be used in an MCache, so it must come after the + // linked list operations above (actually, just after the + // lock of c above.) + atomicstore(&s.sweepgen, mheap_.sweepgen) + + if s.ref != 0 { + unlock(&c.lock) + return false + } + + // s is completely freed, return it to the heap. + mSpanList_Remove(s) + s.needzero = 1 + s.freelist = nil + unlock(&c.lock) + unmarkspan(uintptr(s.start)<<_PageShift, s.npages<<_PageShift) + mHeap_Free(&mheap_, s, 0) + return true +} + +// Fetch a new span from the heap and carve into objects for the free list. +func mCentral_Grow(c *mcentral) *mspan { + npages := uintptr(class_to_allocnpages[c.sizeclass]) + size := uintptr(class_to_size[c.sizeclass]) + n := (npages << _PageShift) / size + + s := mHeap_Alloc(&mheap_, npages, c.sizeclass, false, true) + if s == nil { + return nil + } + + // Carve span into sequence of blocks. + tailp := &s.freelist + p := uintptr(s.start << _PageShift) + s.limit = p + size*n + for i := uintptr(0); i < n; i++ { + v := (*mlink)(unsafe.Pointer(p)) + *tailp = v + tailp = &v.next + p += size + } + *tailp = nil + markspan(unsafe.Pointer(uintptr(s.start)<<_PageShift), size, n, size*n < s.npages<<_PageShift) + return s +} diff --git a/src/runtime/mem.go b/src/runtime/mem.go index e6f1eb0e6..183567251 100644 --- a/src/runtime/mem.go +++ b/src/runtime/mem.go @@ -59,7 +59,11 @@ type MemStats struct { } } -var sizeof_C_MStats uintptr // filled in by malloc.goc +// Size of the trailing by_size array differs between Go and C, +// and all data after by_size is local to runtime, not exported. +// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. +// sizeof_C_MStats is what C thinks about size of Go struct. +var sizeof_C_MStats = unsafe.Offsetof(memstats.by_size) + 61*unsafe.Sizeof(memstats.by_size[0]) func init() { var memStats MemStats @@ -78,15 +82,16 @@ func ReadMemStats(m *MemStats) { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) - gp.m.ptrarg[0] = noescape(unsafe.Pointer(m)) - onM(readmemstats_m) + systemstack(func() { + readmemstats_m(m) + }) gp.m.gcing = 0 gp.m.locks++ semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) gp.m.locks-- } @@ -95,14 +100,15 @@ func writeHeapDump(fd uintptr) { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) - gp.m.scalararg[0] = fd - onM(writeheapdump_m) + systemstack(func() { + writeheapdump_m(fd) + }) gp.m.gcing = 0 gp.m.locks++ semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) gp.m.locks-- } diff --git a/src/runtime/mem_bsd.go b/src/runtime/mem_bsd.go new file mode 100644 index 000000000..4bd40a39f --- /dev/null +++ b/src/runtime/mem_bsd.go @@ -0,0 +1,88 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly freebsd netbsd openbsd solaris + +package runtime + +import "unsafe" + +//go:nosplit +func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer { + v := unsafe.Pointer(mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)) + if uintptr(v) < 4096 { + return nil + } + xadd64(stat, int64(n)) + return v +} + +func sysUnused(v unsafe.Pointer, n uintptr) { + madvise(v, n, _MADV_FREE) +} + +func sysUsed(v unsafe.Pointer, n uintptr) { +} + +func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) { + xadd64(stat, -int64(n)) + munmap(v, n) +} + +func sysFault(v unsafe.Pointer, n uintptr) { + mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) +} + +func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer { + // On 64-bit, people with ulimit -v set complain if we reserve too + // much address space. Instead, assume that the reservation is okay + // and check the assumption in SysMap. + if ptrSize == 8 && uint64(n) > 1<<32 { + *reserved = false + return v + } + + p := unsafe.Pointer(mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)) + if uintptr(p) < 4096 { + return nil + } + *reserved = true + return p +} + +func sysMap(v unsafe.Pointer, n uintptr, reserved bool, stat *uint64) { + const _ENOMEM = 12 + + xadd64(stat, int64(n)) + + // On 64-bit, we don't actually have v reserved, so tread carefully. + if !reserved { + flags := int32(_MAP_ANON | _MAP_PRIVATE) + if GOOS == "dragonfly" { + // TODO(jsing): For some reason DragonFly seems to return + // memory at a different address than we requested, even when + // there should be no reason for it to do so. This can be + // avoided by using MAP_FIXED, but I'm not sure we should need + // to do this - we do not on other platforms. + flags |= _MAP_FIXED + } + p := mmap(v, n, _PROT_READ|_PROT_WRITE, flags, -1, 0) + if uintptr(p) == _ENOMEM { + gothrow("runtime: out of memory") + } + if p != v { + print("runtime: address space conflict: map(", v, ") = ", p, "\n") + gothrow("runtime: address space conflict") + } + return + } + + p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) + if uintptr(p) == _ENOMEM { + gothrow("runtime: out of memory") + } + if p != v { + gothrow("runtime: cannot map pages in arena address space") + } +} diff --git a/src/runtime/mem_darwin.c b/src/runtime/mem_darwin.c deleted file mode 100644 index bf3ede577..000000000 --- a/src/runtime/mem_darwin.c +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - // Linux's MADV_DONTNEED is like BSD's MADV_FREE. - runtime·madvise(v, n, MADV_FREE); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - *reserved = true; - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - return p; -} - -enum -{ - ENOMEM = 12, -}; - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - USED(reserved); - - runtime·xadd64(stat, n); - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_darwin.go b/src/runtime/mem_darwin.go new file mode 100644 index 000000000..1bee933d0 --- /dev/null +++ b/src/runtime/mem_darwin.go @@ -0,0 +1,58 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:nosplit +func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer { + v := (unsafe.Pointer)(mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)) + if uintptr(v) < 4096 { + return nil + } + xadd64(stat, int64(n)) + return v +} + +func sysUnused(v unsafe.Pointer, n uintptr) { + // Linux's MADV_DONTNEED is like BSD's MADV_FREE. + madvise(v, n, _MADV_FREE) +} + +func sysUsed(v unsafe.Pointer, n uintptr) { +} + +func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) { + xadd64(stat, -int64(n)) + munmap(v, n) +} + +func sysFault(v unsafe.Pointer, n uintptr) { + mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) +} + +func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer { + *reserved = true + p := (unsafe.Pointer)(mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)) + if uintptr(p) < 4096 { + return nil + } + return p +} + +const ( + _ENOMEM = 12 +) + +func sysMap(v unsafe.Pointer, n uintptr, reserved bool, stat *uint64) { + xadd64(stat, int64(n)) + p := (unsafe.Pointer)(mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)) + if uintptr(p) == _ENOMEM { + gothrow("runtime: out of memory") + } + if p != v { + gothrow("runtime: cannot map pages in arena address space") + } +} diff --git a/src/runtime/mem_dragonfly.c b/src/runtime/mem_dragonfly.c deleted file mode 100644 index 11457b2c0..000000000 --- a/src/runtime/mem_dragonfly.c +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - ENOMEM = 12, -}; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - runtime·madvise(v, n, MADV_FREE); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // and check the assumption in SysMap. - if(sizeof(void*) == 8 && n > 1LL<<32) { - *reserved = false; - return v; - } - - *reserved = true; - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - // TODO(jsing): For some reason DragonFly seems to return - // memory at a different address than we requested, even when - // there should be no reason for it to do so. This can be - // avoided by using MAP_FIXED, but I'm not sure we should need - // to do this - we do not on other platforms. - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_freebsd.c b/src/runtime/mem_freebsd.c deleted file mode 100644 index 18a9a2f5b..000000000 --- a/src/runtime/mem_freebsd.c +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - ENOMEM = 12, -}; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - runtime·madvise(v, n, MADV_FREE); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // and check the assumption in SysMap. - if(sizeof(void*) == 8 && n > 1LL<<32) { - *reserved = false; - return v; - } - - *reserved = true; - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_linux.c b/src/runtime/mem_linux.c deleted file mode 100644 index bfb405607..000000000 --- a/src/runtime/mem_linux.c +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - _PAGE_SIZE = 4096, - EACCES = 13, -}; - -static int32 -addrspace_free(void *v, uintptr n) -{ - int32 errval; - uintptr chunk; - uintptr off; - - // NOTE: vec must be just 1 byte long here. - // Mincore returns ENOMEM if any of the pages are unmapped, - // but we want to know that all of the pages are unmapped. - // To make these the same, we can only ask about one page - // at a time. See golang.org/issue/7476. - static byte vec[1]; - - for(off = 0; off < n; off += chunk) { - chunk = _PAGE_SIZE * sizeof vec; - if(chunk > (n - off)) - chunk = n - off; - errval = runtime·mincore((int8*)v + off, chunk, vec); - // ENOMEM means unmapped, which is what we want. - // Anything else we assume means the pages are mapped. - if (errval != -ENOMEM) - return 0; - } - return 1; -} - -static void * -mmap_fixed(byte *v, uintptr n, int32 prot, int32 flags, int32 fd, uint32 offset) -{ - void *p; - - p = runtime·mmap(v, n, prot, flags, fd, offset); - if(p != v && addrspace_free(v, n)) { - // On some systems, mmap ignores v without - // MAP_FIXED, so retry if the address space is free. - if(p > (void*)4096) - runtime·munmap(p, n); - p = runtime·mmap(v, n, prot, flags|MAP_FIXED, fd, offset); - } - return p; -} - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *p; - - p = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) { - if(p == (void*)EACCES) { - runtime·printf("runtime: mmap: access denied\n"); - runtime·printf("if you're running SELinux, enable execmem for this process.\n"); - runtime·exit(2); - } - if(p == (void*)EAGAIN) { - runtime·printf("runtime: mmap: too much locked memory (check 'ulimit -l').\n"); - runtime·exit(2); - } - return nil; - } - runtime·xadd64(stat, n); - return p; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - runtime·madvise(v, n, MADV_DONTNEED); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // if we can reserve at least 64K and check the assumption in SysMap. - // Only user-mode Linux (UML) rejects these requests. - if(sizeof(void*) == 8 && n > 1LL<<32) { - p = mmap_fixed(v, 64<<10, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if (p != v) { - if(p >= (void*)4096) - runtime·munmap(p, 64<<10); - return nil; - } - runtime·munmap(p, 64<<10); - *reserved = false; - return v; - } - - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if((uintptr)p < 4096) - return nil; - *reserved = true; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - p = mmap_fixed(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go new file mode 100644 index 000000000..85b55ef49 --- /dev/null +++ b/src/runtime/mem_linux.go @@ -0,0 +1,135 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + _PAGE_SIZE = _PhysPageSize + _EACCES = 13 +) + +// NOTE: vec must be just 1 byte long here. +// Mincore returns ENOMEM if any of the pages are unmapped, +// but we want to know that all of the pages are unmapped. +// To make these the same, we can only ask about one page +// at a time. See golang.org/issue/7476. +var addrspace_vec [1]byte + +func addrspace_free(v unsafe.Pointer, n uintptr) bool { + var chunk uintptr + for off := uintptr(0); off < n; off += chunk { + chunk = _PAGE_SIZE * uintptr(len(addrspace_vec)) + if chunk > (n - off) { + chunk = n - off + } + errval := mincore(unsafe.Pointer(uintptr(v)+off), chunk, &addrspace_vec[0]) + // ENOMEM means unmapped, which is what we want. + // Anything else we assume means the pages are mapped. + if errval != -_ENOMEM { + return false + } + } + return true +} + +func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uint32) unsafe.Pointer { + p := mmap(v, n, prot, flags, fd, offset) + // On some systems, mmap ignores v without + // MAP_FIXED, so retry if the address space is free. + if p != v && addrspace_free(v, n) { + if uintptr(p) > 4096 { + munmap(p, n) + } + p = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset) + } + return p +} + +//go:nosplit +func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer { + p := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) + if uintptr(p) < 4096 { + if uintptr(p) == _EACCES { + print("runtime: mmap: access denied\n") + print("if you're running SELinux, enable execmem for this process.\n") + exit(2) + } + if uintptr(p) == _EAGAIN { + print("runtime: mmap: too much locked memory (check 'ulimit -l').\n") + exit(2) + } + return nil + } + xadd64(stat, int64(n)) + return p +} + +func sysUnused(v unsafe.Pointer, n uintptr) { + madvise(v, n, _MADV_DONTNEED) +} + +func sysUsed(v unsafe.Pointer, n uintptr) { +} + +func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) { + xadd64(stat, -int64(n)) + munmap(v, n) +} + +func sysFault(v unsafe.Pointer, n uintptr) { + mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) +} + +func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer { + // On 64-bit, people with ulimit -v set complain if we reserve too + // much address space. Instead, assume that the reservation is okay + // if we can reserve at least 64K and check the assumption in SysMap. + // Only user-mode Linux (UML) rejects these requests. + if ptrSize == 7 && uint64(n) > 1<<32 { + p := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) + if p != v { + if uintptr(p) >= 4096 { + munmap(p, 64<<10) + } + return nil + } + munmap(p, 64<<10) + *reserved = false + return v + } + + p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) + if uintptr(p) < 4096 { + return nil + } + *reserved = true + return p +} + +func sysMap(v unsafe.Pointer, n uintptr, reserved bool, stat *uint64) { + xadd64(stat, int64(n)) + + // On 64-bit, we don't actually have v reserved, so tread carefully. + if !reserved { + p := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) + if uintptr(p) == _ENOMEM { + gothrow("runtime: out of memory") + } + if p != v { + print("runtime: address space conflict: map(", v, ") = ", p, "\n") + gothrow("runtime: address space conflict") + } + return + } + + p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) + if uintptr(p) == _ENOMEM { + gothrow("runtime: out of memory") + } + if p != v { + gothrow("runtime: cannot map pages in arena address space") + } +} diff --git a/src/runtime/mem_netbsd.c b/src/runtime/mem_netbsd.c deleted file mode 100644 index 31820e517..000000000 --- a/src/runtime/mem_netbsd.c +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - ENOMEM = 12, -}; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - runtime·madvise(v, n, MADV_FREE); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // and check the assumption in SysMap. - if(sizeof(void*) == 8 && n > 1LL<<32) { - *reserved = false; - return v; - } - - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - *reserved = true; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_openbsd.c b/src/runtime/mem_openbsd.c deleted file mode 100644 index 31820e517..000000000 --- a/src/runtime/mem_openbsd.c +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - ENOMEM = 12, -}; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - runtime·madvise(v, n, MADV_FREE); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // and check the assumption in SysMap. - if(sizeof(void*) == 8 && n > 1LL<<32) { - *reserved = false; - return v; - } - - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - *reserved = true; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_solaris.c b/src/runtime/mem_solaris.c deleted file mode 100644 index 8e90ba1d9..000000000 --- a/src/runtime/mem_solaris.c +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "malloc.h" -#include "textflag.h" - -enum -{ - ENOMEM = 12, -}; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - void *v; - - v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) - return nil; - runtime·xadd64(stat, n); - return v; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - USED(v); - USED(n); -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, -(uint64)n); - runtime·munmap(v, n); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - void *p; - - // On 64-bit, people with ulimit -v set complain if we reserve too - // much address space. Instead, assume that the reservation is okay - // and check the assumption in SysMap. - if(sizeof(void*) == 8 && n > 1LL<<32) { - *reserved = false; - return v; - } - - p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p < (void*)4096) - return nil; - *reserved = true; - return p; -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - runtime·xadd64(stat, n); - - // On 64-bit, we don't actually have v reserved, so tread carefully. - if(!reserved) { - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) { - runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); - runtime·throw("runtime: address space conflict"); - } - return; - } - - p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)ENOMEM) - runtime·throw("runtime: out of memory"); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_windows.c b/src/runtime/mem_windows.c deleted file mode 100644 index 6ea992020..000000000 --- a/src/runtime/mem_windows.c +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "os_GOOS.h" -#include "defs_GOOS_GOARCH.h" -#include "malloc.h" -#include "textflag.h" - -enum { - MEM_COMMIT = 0x1000, - MEM_RESERVE = 0x2000, - MEM_DECOMMIT = 0x4000, - MEM_RELEASE = 0x8000, - - PAGE_READWRITE = 0x0004, - PAGE_NOACCESS = 0x0001, -}; - -#pragma dynimport runtime·VirtualAlloc VirtualAlloc "kernel32.dll" -#pragma dynimport runtime·VirtualFree VirtualFree "kernel32.dll" -#pragma dynimport runtime·VirtualProtect VirtualProtect "kernel32.dll" -extern void *runtime·VirtualAlloc; -extern void *runtime·VirtualFree; -extern void *runtime·VirtualProtect; - -#pragma textflag NOSPLIT -void* -runtime·sysAlloc(uintptr n, uint64 *stat) -{ - runtime·xadd64(stat, n); - return runtime·stdcall4(runtime·VirtualAlloc, 0, n, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE); -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - void *r; - uintptr small; - - r = runtime·stdcall3(runtime·VirtualFree, (uintptr)v, n, MEM_DECOMMIT); - if(r != nil) - return; - - // Decommit failed. Usual reason is that we've merged memory from two different - // VirtualAlloc calls, and Windows will only let each VirtualFree handle pages from - // a single VirtualAlloc. It is okay to specify a subset of the pages from a single alloc, - // just not pages from multiple allocs. This is a rare case, arising only when we're - // trying to give memory back to the operating system, which happens on a time - // scale of minutes. It doesn't have to be terribly fast. Instead of extra bookkeeping - // on all our VirtualAlloc calls, try freeing successively smaller pieces until - // we manage to free something, and then repeat. This ends up being O(n log n) - // in the worst case, but that's fast enough. - while(n > 0) { - small = n; - while(small >= 4096 && runtime·stdcall3(runtime·VirtualFree, (uintptr)v, small, MEM_DECOMMIT) == nil) - small = (small / 2) & ~(4096-1); - if(small < 4096) - runtime·throw("runtime: failed to decommit pages"); - v = (byte*)v + small; - n -= small; - } -} - -void -runtime·SysUsed(void *v, uintptr n) -{ - void *r; - uintptr small; - - r = runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, n, MEM_COMMIT, PAGE_READWRITE); - if(r != v) - runtime·throw("runtime: failed to commit pages"); - - // Commit failed. See SysUnused. - while(n > 0) { - small = n; - while(small >= 4096 && runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, small, MEM_COMMIT, PAGE_READWRITE) == nil) - small = (small / 2) & ~(4096-1); - if(small < 4096) - runtime·throw("runtime: failed to decommit pages"); - v = (byte*)v + small; - n -= small; - } -} - -void -runtime·SysFree(void *v, uintptr n, uint64 *stat) -{ - uintptr r; - - runtime·xadd64(stat, -(uint64)n); - r = (uintptr)runtime·stdcall3(runtime·VirtualFree, (uintptr)v, 0, MEM_RELEASE); - if(r == 0) - runtime·throw("runtime: failed to release pages"); -} - -void -runtime·SysFault(void *v, uintptr n) -{ - // SysUnused makes the memory inaccessible and prevents its reuse - runtime·SysUnused(v, n); -} - -void* -runtime·SysReserve(void *v, uintptr n, bool *reserved) -{ - *reserved = true; - // v is just a hint. - // First try at v. - v = runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, n, MEM_RESERVE, PAGE_READWRITE); - if(v != nil) - return v; - - // Next let the kernel choose the address. - return runtime·stdcall4(runtime·VirtualAlloc, 0, n, MEM_RESERVE, PAGE_READWRITE); -} - -void -runtime·SysMap(void *v, uintptr n, bool reserved, uint64 *stat) -{ - void *p; - - USED(reserved); - - runtime·xadd64(stat, n); - p = runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, n, MEM_COMMIT, PAGE_READWRITE); - if(p != v) - runtime·throw("runtime: cannot map pages in arena address space"); -} diff --git a/src/runtime/mem_windows.go b/src/runtime/mem_windows.go new file mode 100644 index 000000000..a1dcad013 --- /dev/null +++ b/src/runtime/mem_windows.go @@ -0,0 +1,119 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +const ( + _MEM_COMMIT = 0x1000 + _MEM_RESERVE = 0x2000 + _MEM_DECOMMIT = 0x4000 + _MEM_RELEASE = 0x8000 + + _PAGE_READWRITE = 0x0004 + _PAGE_NOACCESS = 0x0001 +) + +//go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc "kernel32.dll" +//go:cgo_import_dynamic runtime._VirtualFree VirtualFree "kernel32.dll" +//go:cgo_import_dynamic runtime._VirtualProtect VirtualProtect "kernel32.dll" + +var ( + _VirtualAlloc, + _VirtualFree, + _VirtualProtect stdFunction +) + +//go:nosplit +func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer { + xadd64(stat, int64(n)) + return unsafe.Pointer(stdcall4(_VirtualAlloc, 0, n, _MEM_COMMIT|_MEM_RESERVE, _PAGE_READWRITE)) +} + +func sysUnused(v unsafe.Pointer, n uintptr) { + r := stdcall3(_VirtualFree, uintptr(v), n, _MEM_DECOMMIT) + if r != 0 { + return + } + + // Decommit failed. Usual reason is that we've merged memory from two different + // VirtualAlloc calls, and Windows will only let each VirtualFree handle pages from + // a single VirtualAlloc. It is okay to specify a subset of the pages from a single alloc, + // just not pages from multiple allocs. This is a rare case, arising only when we're + // trying to give memory back to the operating system, which happens on a time + // scale of minutes. It doesn't have to be terribly fast. Instead of extra bookkeeping + // on all our VirtualAlloc calls, try freeing successively smaller pieces until + // we manage to free something, and then repeat. This ends up being O(n log n) + // in the worst case, but that's fast enough. + for n > 0 { + small := n + for small >= 4096 && stdcall3(_VirtualFree, uintptr(v), small, _MEM_DECOMMIT) == 0 { + small /= 2 + small &^= 4096 - 1 + } + if small < 4096 { + gothrow("runtime: failed to decommit pages") + } + v = add(v, small) + n -= small + } +} + +func sysUsed(v unsafe.Pointer, n uintptr) { + r := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE) + if r != uintptr(v) { + gothrow("runtime: failed to commit pages") + } + + // Commit failed. See SysUnused. + for n > 0 { + small := n + for small >= 4096 && stdcall4(_VirtualAlloc, uintptr(v), small, _MEM_COMMIT, _PAGE_READWRITE) == 0 { + small /= 2 + small &^= 4096 - 1 + } + if small < 4096 { + gothrow("runtime: failed to decommit pages") + } + v = add(v, small) + n -= small + } +} + +func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) { + xadd64(stat, -int64(n)) + r := stdcall3(_VirtualFree, uintptr(v), 0, _MEM_RELEASE) + if r == 0 { + gothrow("runtime: failed to release pages") + } +} + +func sysFault(v unsafe.Pointer, n uintptr) { + // SysUnused makes the memory inaccessible and prevents its reuse + sysUnused(v, n) +} + +func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer { + *reserved = true + // v is just a hint. + // First try at v. + v = unsafe.Pointer(stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_RESERVE, _PAGE_READWRITE)) + if v != nil { + return v + } + + // Next let the kernel choose the address. + return unsafe.Pointer(stdcall4(_VirtualAlloc, 0, n, _MEM_RESERVE, _PAGE_READWRITE)) +} + +func sysMap(v unsafe.Pointer, n uintptr, reserved bool, stat *uint64) { + xadd64(stat, int64(n)) + p := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE) + if p != uintptr(v) { + gothrow("runtime: cannot map pages in arena address space") + } +} diff --git a/src/runtime/memclr_386.s b/src/runtime/memclr_386.s index 1520aea2e..3f20b69c8 100644 --- a/src/runtime/memclr_386.s +++ b/src/runtime/memclr_386.s @@ -15,31 +15,31 @@ TEXT runtime·memclr(SB), NOSPLIT, $0-8 XORL AX, AX // MOVOU seems always faster than REP STOSL. -clr_tail: +tail: TESTL BX, BX - JEQ clr_0 + JEQ _0 CMPL BX, $2 - JBE clr_1or2 + JBE _1or2 CMPL BX, $4 - JBE clr_3or4 + JBE _3or4 CMPL BX, $8 - JBE clr_5through8 + JBE _5through8 CMPL BX, $16 - JBE clr_9through16 + JBE _9through16 TESTL $0x4000000, runtime·cpuid_edx(SB) // check for sse2 JEQ nosse2 PXOR X0, X0 CMPL BX, $32 - JBE clr_17through32 + JBE _17through32 CMPL BX, $64 - JBE clr_33through64 + JBE _33through64 CMPL BX, $128 - JBE clr_65through128 + JBE _65through128 CMPL BX, $256 - JBE clr_129through256 + JBE _129through256 // TODO: use branch table and BSR to make this just a single dispatch -clr_loop: +loop: MOVOU X0, 0(DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) @@ -59,40 +59,40 @@ clr_loop: SUBL $256, BX ADDL $256, DI CMPL BX, $256 - JAE clr_loop - JMP clr_tail + JAE loop + JMP tail -clr_1or2: +_1or2: MOVB AX, (DI) MOVB AX, -1(DI)(BX*1) RET -clr_0: +_0: RET -clr_3or4: +_3or4: MOVW AX, (DI) MOVW AX, -2(DI)(BX*1) RET -clr_5through8: +_5through8: MOVL AX, (DI) MOVL AX, -4(DI)(BX*1) RET -clr_9through16: +_9through16: MOVL AX, (DI) MOVL AX, 4(DI) MOVL AX, -8(DI)(BX*1) MOVL AX, -4(DI)(BX*1) RET -clr_17through32: +_17through32: MOVOU X0, (DI) MOVOU X0, -16(DI)(BX*1) RET -clr_33through64: +_33through64: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, -32(DI)(BX*1) MOVOU X0, -16(DI)(BX*1) RET -clr_65through128: +_65through128: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) @@ -102,7 +102,7 @@ clr_65through128: MOVOU X0, -32(DI)(BX*1) MOVOU X0, -16(DI)(BX*1) RET -clr_129through256: +_129through256: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) @@ -126,5 +126,5 @@ nosse2: REP STOSL ANDL $3, BX - JNE clr_tail + JNE tail RET diff --git a/src/runtime/memclr_amd64.s b/src/runtime/memclr_amd64.s index 94a2c7f23..ec24f1db2 100644 --- a/src/runtime/memclr_amd64.s +++ b/src/runtime/memclr_amd64.s @@ -15,30 +15,30 @@ TEXT runtime·memclr(SB), NOSPLIT, $0-16 XORQ AX, AX // MOVOU seems always faster than REP STOSQ. -clr_tail: +tail: TESTQ BX, BX - JEQ clr_0 + JEQ _0 CMPQ BX, $2 - JBE clr_1or2 + JBE _1or2 CMPQ BX, $4 - JBE clr_3or4 + JBE _3or4 CMPQ BX, $8 - JBE clr_5through8 + JBE _5through8 CMPQ BX, $16 - JBE clr_9through16 + JBE _9through16 PXOR X0, X0 CMPQ BX, $32 - JBE clr_17through32 + JBE _17through32 CMPQ BX, $64 - JBE clr_33through64 + JBE _33through64 CMPQ BX, $128 - JBE clr_65through128 + JBE _65through128 CMPQ BX, $256 - JBE clr_129through256 + JBE _129through256 // TODO: use branch table and BSR to make this just a single dispatch // TODO: for really big clears, use MOVNTDQ. -clr_loop: +loop: MOVOU X0, 0(DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) @@ -58,38 +58,38 @@ clr_loop: SUBQ $256, BX ADDQ $256, DI CMPQ BX, $256 - JAE clr_loop - JMP clr_tail + JAE loop + JMP tail -clr_1or2: +_1or2: MOVB AX, (DI) MOVB AX, -1(DI)(BX*1) RET -clr_0: +_0: RET -clr_3or4: +_3or4: MOVW AX, (DI) MOVW AX, -2(DI)(BX*1) RET -clr_5through8: +_5through8: MOVL AX, (DI) MOVL AX, -4(DI)(BX*1) RET -clr_9through16: +_9through16: MOVQ AX, (DI) MOVQ AX, -8(DI)(BX*1) RET -clr_17through32: +_17through32: MOVOU X0, (DI) MOVOU X0, -16(DI)(BX*1) RET -clr_33through64: +_33through64: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, -32(DI)(BX*1) MOVOU X0, -16(DI)(BX*1) RET -clr_65through128: +_65through128: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) @@ -99,7 +99,7 @@ clr_65through128: MOVOU X0, -32(DI)(BX*1) MOVOU X0, -16(DI)(BX*1) RET -clr_129through256: +_129through256: MOVOU X0, (DI) MOVOU X0, 16(DI) MOVOU X0, 32(DI) diff --git a/src/runtime/memclr_plan9_386.s b/src/runtime/memclr_plan9_386.s index b4b671f77..50f327b4e 100644 --- a/src/runtime/memclr_plan9_386.s +++ b/src/runtime/memclr_plan9_386.s @@ -10,40 +10,40 @@ TEXT runtime·memclr(SB), NOSPLIT, $0-8 MOVL n+4(FP), BX XORL AX, AX -clr_tail: +tail: TESTL BX, BX - JEQ clr_0 + JEQ _0 CMPL BX, $2 - JBE clr_1or2 + JBE _1or2 CMPL BX, $4 - JBE clr_3or4 + JBE _3or4 CMPL BX, $8 - JBE clr_5through8 + JBE _5through8 CMPL BX, $16 - JBE clr_9through16 + JBE _9through16 MOVL BX, CX SHRL $2, CX REP STOSL ANDL $3, BX - JNE clr_tail + JNE tail RET -clr_1or2: +_1or2: MOVB AX, (DI) MOVB AX, -1(DI)(BX*1) RET -clr_0: +_0: RET -clr_3or4: +_3or4: MOVW AX, (DI) MOVW AX, -2(DI)(BX*1) RET -clr_5through8: +_5through8: MOVL AX, (DI) MOVL AX, -4(DI)(BX*1) RET -clr_9through16: +_9through16: MOVL AX, (DI) MOVL AX, 4(DI) MOVL AX, -8(DI)(BX*1) diff --git a/src/runtime/memclr_power64x.s b/src/runtime/memclr_power64x.s new file mode 100644 index 000000000..dfad64b6f --- /dev/null +++ b/src/runtime/memclr_power64x.s @@ -0,0 +1,20 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le + +#include "textflag.h" + +// void runtime·memclr(void*, uintptr) +TEXT runtime·memclr(SB),NOSPLIT,$0-16 + MOVD ptr+0(FP), R3 + MOVD n+8(FP), R4 + CMP R4, $0 + BEQ done + SUB $1, R3 + MOVD R4, CTR + MOVBU R0, 1(R3) + BC 25, 0, -1(PC) // bdnz+ $-4 +done: + RETURN diff --git a/src/runtime/memmove_power64x.s b/src/runtime/memmove_power64x.s new file mode 100644 index 000000000..2b04d8319 --- /dev/null +++ b/src/runtime/memmove_power64x.s @@ -0,0 +1,40 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le + +#include "textflag.h" + +// void runtime·memmove(void*, void*, uintptr) +TEXT runtime·memmove(SB), NOSPLIT, $-8-24 + MOVD to+0(FP), R3 + MOVD from+8(FP), R4 + MOVD n+16(FP), R5 + CMP R5, $0 + BNE check + RETURN + +check: + CMP R3, R4 + BGT backward + + SUB $1, R3 + ADD R3, R5 + SUB $1, R4 +loop: + MOVBU 1(R4), R6 + MOVBU R6, 1(R3) + CMP R3, R5 + BNE loop + RETURN + +backward: + ADD R5, R4 + ADD R3, R5 +loop1: + MOVBU -1(R4), R6 + MOVBU R6, -1(R5) + CMP R3, R5 + BNE loop1 + RETURN diff --git a/src/runtime/mfixalloc.c b/src/runtime/mfixalloc.c deleted file mode 100644 index d670629da..000000000 --- a/src/runtime/mfixalloc.c +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Fixed-size object allocator. Returned memory is not zeroed. -// -// See malloc.h for overview. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -// Initialize f to allocate objects of the given size, -// using the allocator to obtain chunks of memory. -void -runtime·FixAlloc_Init(FixAlloc *f, uintptr size, void (*first)(void*, byte*), void *arg, uint64 *stat) -{ - f->size = size; - f->first = first; - f->arg = arg; - f->list = nil; - f->chunk = nil; - f->nchunk = 0; - f->inuse = 0; - f->stat = stat; -} - -void* -runtime·FixAlloc_Alloc(FixAlloc *f) -{ - void *v; - - if(f->size == 0) { - runtime·printf("runtime: use of FixAlloc_Alloc before FixAlloc_Init\n"); - runtime·throw("runtime: internal error"); - } - - if(f->list) { - v = f->list; - f->list = *(void**)f->list; - f->inuse += f->size; - return v; - } - if(f->nchunk < f->size) { - f->chunk = runtime·persistentalloc(FixAllocChunk, 0, f->stat); - f->nchunk = FixAllocChunk; - } - v = f->chunk; - if(f->first) - f->first(f->arg, v); - f->chunk += f->size; - f->nchunk -= f->size; - f->inuse += f->size; - return v; -} - -void -runtime·FixAlloc_Free(FixAlloc *f, void *p) -{ - f->inuse -= f->size; - *(void**)p = f->list; - f->list = p; -} - diff --git a/src/runtime/mfixalloc.go b/src/runtime/mfixalloc.go new file mode 100644 index 000000000..b66a17e41 --- /dev/null +++ b/src/runtime/mfixalloc.go @@ -0,0 +1,59 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Fixed-size object allocator. Returned memory is not zeroed. +// +// See malloc.h for overview. + +package runtime + +import "unsafe" + +// Initialize f to allocate objects of the given size, +// using the allocator to obtain chunks of memory. +func fixAlloc_Init(f *fixalloc, size uintptr, first func(unsafe.Pointer, unsafe.Pointer), arg unsafe.Pointer, stat *uint64) { + f.size = size + f.first = *(*unsafe.Pointer)(unsafe.Pointer(&first)) + f.arg = arg + f.list = nil + f.chunk = nil + f.nchunk = 0 + f.inuse = 0 + f.stat = stat +} + +func fixAlloc_Alloc(f *fixalloc) unsafe.Pointer { + if f.size == 0 { + print("runtime: use of FixAlloc_Alloc before FixAlloc_Init\n") + gothrow("runtime: internal error") + } + + if f.list != nil { + v := unsafe.Pointer(f.list) + f.list = f.list.next + f.inuse += f.size + return v + } + if uintptr(f.nchunk) < f.size { + f.chunk = (*uint8)(persistentalloc(_FixAllocChunk, 0, f.stat)) + f.nchunk = _FixAllocChunk + } + + v := (unsafe.Pointer)(f.chunk) + if f.first != nil { + fn := *(*func(unsafe.Pointer, unsafe.Pointer))(unsafe.Pointer(&f.first)) + fn(f.arg, v) + } + f.chunk = (*byte)(add(unsafe.Pointer(f.chunk), f.size)) + f.nchunk -= uint32(f.size) + f.inuse += f.size + return v +} + +func fixAlloc_Free(f *fixalloc, p unsafe.Pointer) { + f.inuse -= f.size + v := (*mlink)(p) + v.next = f.list + f.list = v +} diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go new file mode 100644 index 000000000..f44d7ddbc --- /dev/null +++ b/src/runtime/mgc.go @@ -0,0 +1,1798 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(rsc): The code having to do with the heap bitmap needs very serious cleanup. +// It has gotten completely out of control. + +// Garbage collector (GC). +// +// GC is: +// - mark&sweep +// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc) +// - parallel (up to MaxGcproc threads) +// - partially concurrent (mark is stop-the-world, while sweep is concurrent) +// - non-moving/non-compacting +// - full (non-partial) +// +// GC rate. +// Next GC is after we've allocated an extra amount of memory proportional to +// the amount already in use. The proportion is controlled by GOGC environment variable +// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M +// (this mark is tracked in next_gc variable). This keeps the GC cost in linear +// proportion to the allocation cost. Adjusting GOGC just changes the linear constant +// (and also the amount of extra memory used). +// +// Concurrent sweep. +// The sweep phase proceeds concurrently with normal program execution. +// The heap is swept span-by-span both lazily (when a goroutine needs another span) +// and concurrently in a background goroutine (this helps programs that are not CPU bound). +// However, at the end of the stop-the-world GC phase we don't know the size of the live heap, +// and so next_gc calculation is tricky and happens as follows. +// At the end of the stop-the-world phase next_gc is conservatively set based on total +// heap size; all spans are marked as "needs sweeping". +// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory. +// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc +// closer to the target value. However, this is not enough to avoid over-allocating memory. +// Consider that a goroutine wants to allocate a new span for a large object and +// there are no free swept spans, but there are small-object unswept spans. +// If the goroutine naively allocates a new span, it can surpass the yet-unknown +// target next_gc value. In order to prevent such cases (1) when a goroutine needs +// to allocate a new small-object span, it sweeps small-object spans for the same +// object size until it frees at least one object; (2) when a goroutine needs to +// allocate large-object span from heap, it sweeps spans until it frees at least +// that many pages into heap. Together these two measures ensure that we don't surpass +// target next_gc value by a large margin. There is an exception: if a goroutine sweeps +// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span, +// but there can still be other one-page unswept spans which could be combined into a two-page span. +// It's critical to ensure that no operations proceed on unswept spans (that would corrupt +// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache, +// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it. +// When a goroutine explicitly frees an object or sets a finalizer, it ensures that +// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish). +// The finalizer goroutine is kicked off only when all spans are swept. +// When the next GC starts, it sweeps all not-yet-swept spans (if any). + +package runtime + +import "unsafe" + +const ( + _DebugGC = 0 + _DebugGCPtrs = false // if true, print trace of every pointer load during GC + _ConcurrentSweep = true + + _WorkbufSize = 4 * 1024 + _FinBlockSize = 4 * 1024 + _RootData = 0 + _RootBss = 1 + _RootFinalizers = 2 + _RootSpans = 3 + _RootFlushCaches = 4 + _RootCount = 5 +) + +// ptrmask for an allocation containing a single pointer. +var oneptr = [...]uint8{bitsPointer} + +// Initialized from $GOGC. GOGC=off means no gc. +var gcpercent int32 + +// Holding worldsema grants an M the right to try to stop the world. +// The procedure is: +// +// semacquire(&worldsema); +// m.gcing = 1; +// stoptheworld(); +// +// ... do stuff ... +// +// m.gcing = 0; +// semrelease(&worldsema); +// starttheworld(); +// +var worldsema uint32 = 1 + +type workbuf struct { + node lfnode // must be first + nobj uintptr + obj [(_WorkbufSize - unsafe.Sizeof(lfnode{}) - ptrSize) / ptrSize]uintptr +} + +var data, edata, bss, ebss, gcdata, gcbss struct{} + +var finlock mutex // protects the following variables +var fing *g // goroutine that runs finalizers +var finq *finblock // list of finalizers that are to be executed +var finc *finblock // cache of free blocks +var finptrmask [_FinBlockSize / ptrSize / pointersPerByte]byte +var fingwait bool +var fingwake bool +var allfin *finblock // list of all blocks + +var gcdatamask bitvector +var gcbssmask bitvector + +var gclock mutex + +var badblock [1024]uintptr +var nbadblock int32 + +type workdata struct { + full uint64 // lock-free list of full blocks + empty uint64 // lock-free list of empty blocks + pad0 [_CacheLineSize]uint8 // prevents false-sharing between full/empty and nproc/nwait + nproc uint32 + tstart int64 + nwait uint32 + ndone uint32 + alldone note + markfor *parfor + + // Copy of mheap.allspans for marker or sweeper. + spans []*mspan +} + +var work workdata + +//go:linkname weak_cgo_allocate go.weak.runtime._cgo_allocate_internal +var weak_cgo_allocate byte + +// Is _cgo_allocate linked into the binary? +func have_cgo_allocate() bool { + return &weak_cgo_allocate != nil +} + +// scanblock scans a block of n bytes starting at pointer b for references +// to other objects, scanning any it finds recursively until there are no +// unscanned objects left. Instead of using an explicit recursion, it keeps +// a work list in the Workbuf* structures and loops in the main function +// body. Keeping an explicit work list is easier on the stack allocator and +// more efficient. +func scanblock(b, n uintptr, ptrmask *uint8) { + // Cache memory arena parameters in local vars. + arena_start := mheap_.arena_start + arena_used := mheap_.arena_used + + wbuf := getempty(nil) + nobj := wbuf.nobj + wp := &wbuf.obj[nobj] + keepworking := b == 0 + + var ptrbitp unsafe.Pointer + + // ptrmask can have 2 possible values: + // 1. nil - obtain pointer mask from GC bitmap. + // 2. pointer to a compact mask (for stacks and data). + goto_scanobj := b != 0 + + for { + if goto_scanobj { + goto_scanobj = false + } else { + if nobj == 0 { + // Out of work in workbuf. + if !keepworking { + putempty(wbuf) + return + } + + // Refill workbuf from global queue. + wbuf = getfull(wbuf) + if wbuf == nil { + return + } + nobj = wbuf.nobj + if nobj < uintptr(len(wbuf.obj)) { + wp = &wbuf.obj[nobj] + } else { + wp = nil + } + } + + // If another proc wants a pointer, give it some. + if work.nwait > 0 && nobj > 4 && work.full == 0 { + wbuf.nobj = nobj + wbuf = handoff(wbuf) + nobj = wbuf.nobj + if nobj < uintptr(len(wbuf.obj)) { + wp = &wbuf.obj[nobj] + } else { + wp = nil + } + } + + nobj-- + wp = &wbuf.obj[nobj] + b = *wp + n = arena_used - uintptr(b) + ptrmask = nil // use GC bitmap for pointer info + } + + if _DebugGCPtrs { + print("scanblock ", b, " +", hex(n), " ", ptrmask, "\n") + } + + // Find bits of the beginning of the object. + if ptrmask == nil { + off := (uintptr(b) - arena_start) / ptrSize + ptrbitp = unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1) + } + + var i uintptr + for i = 0; i < n; i += ptrSize { + // Find bits for this word. + var bits uintptr + if ptrmask == nil { + // Check if we have reached end of span. + if (uintptr(b)+i)%_PageSize == 0 && + h_spans[(uintptr(b)-arena_start)>>_PageShift] != h_spans[(uintptr(b)+i-arena_start)>>_PageShift] { + break + } + + // Consult GC bitmap. + bits = uintptr(*(*byte)(ptrbitp)) + + if wordsPerBitmapByte != 2 { + gothrow("alg doesn't work for wordsPerBitmapByte != 2") + } + j := (uintptr(b) + i) / ptrSize & 1 + ptrbitp = add(ptrbitp, -j) + bits >>= gcBits * j + + if bits&bitBoundary != 0 && i != 0 { + break // reached beginning of the next object + } + bits = (bits >> 2) & bitsMask + if bits == bitsDead { + break // reached no-scan part of the object + } + } else { + // dense mask (stack or data) + bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * bitsPerPointer)) & bitsMask + } + + if bits <= _BitsScalar { // BitsScalar || BitsDead + continue + } + + if bits != _BitsPointer { + gothrow("unexpected garbage collection bits") + } + + obj := *(*uintptr)(unsafe.Pointer(b + i)) + obj0 := obj + + markobj: + var s *mspan + var off, bitp, shift, xbits uintptr + + // At this point we have extracted the next potential pointer. + // Check if it points into heap. + if obj == 0 { + continue + } + if obj < arena_start || arena_used <= obj { + if uintptr(obj) < _PhysPageSize && invalidptr != 0 { + s = nil + goto badobj + } + continue + } + + // Mark the object. + obj &^= ptrSize - 1 + off = (obj - arena_start) / ptrSize + bitp = arena_start - off/wordsPerBitmapByte - 1 + shift = (off % wordsPerBitmapByte) * gcBits + xbits = uintptr(*(*byte)(unsafe.Pointer(bitp))) + bits = (xbits >> shift) & bitMask + if (bits & bitBoundary) == 0 { + // Not a beginning of a block, consult span table to find the block beginning. + k := pageID(obj >> _PageShift) + x := k + x -= pageID(arena_start >> _PageShift) + s = h_spans[x] + if s == nil || k < s.start || s.limit <= obj || s.state != mSpanInUse { + // Stack pointers lie within the arena bounds but are not part of the GC heap. + // Ignore them. + if s != nil && s.state == _MSpanStack { + continue + } + goto badobj + } + p := uintptr(s.start) << _PageShift + if s.sizeclass != 0 { + size := s.elemsize + idx := (obj - p) / size + p = p + idx*size + } + if p == obj { + print("runtime: failed to find block beginning for ", hex(p), " s=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), "\n") + gothrow("failed to find block beginning") + } + obj = p + goto markobj + } + + if _DebugGCPtrs { + print("scan *", hex(b+i), " = ", hex(obj0), " => base ", hex(obj), "\n") + } + + if nbadblock > 0 && obj == badblock[nbadblock-1] { + // Running garbage collection again because + // we want to find the path from a root to a bad pointer. + // Found possible next step; extend or finish path. + for j := int32(0); j < nbadblock; j++ { + if badblock[j] == b { + goto AlreadyBad + } + } + print("runtime: found *(", hex(b), "+", hex(i), ") = ", hex(obj0), "+", hex(obj-obj0), "\n") + if ptrmask != nil { + gothrow("bad pointer") + } + if nbadblock >= int32(len(badblock)) { + gothrow("badblock trace too long") + } + badblock[nbadblock] = uintptr(b) + nbadblock++ + AlreadyBad: + } + + // Now we have bits, bitp, and shift correct for + // obj pointing at the base of the object. + // Only care about not marked objects. + if bits&bitMarked != 0 { + continue + } + + // If obj size is greater than 8, then each byte of GC bitmap + // contains info for at most one object. In such case we use + // non-atomic byte store to mark the object. This can lead + // to double enqueue of the object for scanning, but scanning + // is an idempotent operation, so it is OK. This cannot lead + // to bitmap corruption because the single marked bit is the + // only thing that can change in the byte. + // For 8-byte objects we use non-atomic store, if the other + // quadruple is already marked. Otherwise we resort to CAS + // loop for marking. + if xbits&(bitMask|bitMask<<gcBits) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 { + *(*byte)(unsafe.Pointer(bitp)) = uint8(xbits | bitMarked<<shift) + } else { + atomicor8((*byte)(unsafe.Pointer(bitp)), bitMarked<<shift) + } + + if (xbits>>(shift+2))&bitsMask == bitsDead { + continue // noscan object + } + + // Queue the obj for scanning. + // TODO: PREFETCH here. + + // If workbuf is full, obtain an empty one. + if nobj >= uintptr(len(wbuf.obj)) { + wbuf.nobj = nobj + wbuf = getempty(wbuf) + nobj = wbuf.nobj + wp = &wbuf.obj[nobj] + } + *wp = obj + nobj++ + if nobj < uintptr(len(wbuf.obj)) { + wp = &wbuf.obj[nobj] + } else { + wp = nil + } + continue + + badobj: + // If cgo_allocate is linked into the binary, it can allocate + // memory as []unsafe.Pointer that may not contain actual + // pointers and must be scanned conservatively. + // In this case alone, allow the bad pointer. + if have_cgo_allocate() && ptrmask == nil { + continue + } + + // Anything else indicates a bug somewhere. + // If we're in the middle of chasing down a different bad pointer, + // don't confuse the trace by printing about this one. + if nbadblock > 0 { + continue + } + + print("runtime: garbage collector found invalid heap pointer *(", hex(b), "+", hex(i), ")=", hex(obj)) + if s == nil { + print(" s=nil\n") + } else { + print(" span=", uintptr(s.start)<<_PageShift, "-", s.limit, "-", (uintptr(s.start)+s.npages)<<_PageShift, " state=", s.state, "\n") + } + if ptrmask != nil { + gothrow("invalid heap pointer") + } + // Add to badblock list, which will cause the garbage collection + // to keep repeating until it has traced the chain of pointers + // leading to obj all the way back to a root. + if nbadblock == 0 { + badblock[nbadblock] = uintptr(b) + nbadblock++ + } + } + if _DebugGCPtrs { + print("end scanblock ", hex(b), " +", hex(n), " ", ptrmask, "\n") + } + if _DebugGC > 0 && ptrmask == nil { + // For heap objects ensure that we did not overscan. + var p, n uintptr + if mlookup(b, &p, &n, nil) == 0 || b != p || i > n { + print("runtime: scanned (", hex(b), "+", hex(i), "), heap object (", hex(p), "+", hex(n), ")\n") + gothrow("scanblock: scanned invalid object") + } + } + } +} + +func markroot(desc *parfor, i uint32) { + // Note: if you add a case here, please also update heapdump.c:dumproots. + switch i { + case _RootData: + scanblock(uintptr(unsafe.Pointer(&data)), uintptr(unsafe.Pointer(&edata))-uintptr(unsafe.Pointer(&data)), gcdatamask.bytedata) + + case _RootBss: + scanblock(uintptr(unsafe.Pointer(&bss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss)), gcbssmask.bytedata) + + case _RootFinalizers: + for fb := allfin; fb != nil; fb = fb.alllink { + scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), uintptr(fb.cnt)*unsafe.Sizeof(fb.fin[0]), &finptrmask[0]) + } + + case _RootSpans: + // mark MSpan.specials + sg := mheap_.sweepgen + for spanidx := uint32(0); spanidx < uint32(len(work.spans)); spanidx++ { + s := work.spans[spanidx] + if s.state != mSpanInUse { + continue + } + if s.sweepgen != sg { + print("sweep ", s.sweepgen, " ", sg, "\n") + gothrow("gc: unswept span") + } + for sp := s.specials; sp != nil; sp = sp.next { + if sp.kind != _KindSpecialFinalizer { + continue + } + // don't mark finalized object, but scan it so we + // retain everything it points to. + spf := (*specialfinalizer)(unsafe.Pointer(sp)) + // A finalizer can be set for an inner byte of an object, find object beginning. + p := uintptr(s.start<<_PageShift) + uintptr(spf.special.offset)/s.elemsize*s.elemsize + scanblock(p, s.elemsize, nil) + scanblock(uintptr(unsafe.Pointer(&spf.fn)), ptrSize, &oneptr[0]) + } + } + + case _RootFlushCaches: + flushallmcaches() + + default: + // the rest is scanning goroutine stacks + if uintptr(i-_RootCount) >= allglen { + gothrow("markroot: bad index") + } + gp := allgs[i-_RootCount] + // remember when we've first observed the G blocked + // needed only to output in traceback + status := readgstatus(gp) + if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 { + gp.waitsince = work.tstart + } + // Shrink a stack if not much of it is being used. + shrinkstack(gp) + if readgstatus(gp) == _Gdead { + gp.gcworkdone = true + } else { + gp.gcworkdone = false + } + restart := stopg(gp) + scanstack(gp) + if restart { + restartg(gp) + } + } +} + +// Get an empty work buffer off the work.empty list, +// allocating new buffers as needed. +func getempty(b *workbuf) *workbuf { + _g_ := getg() + if b != nil { + lfstackpush(&work.full, &b.node) + } + b = nil + c := _g_.m.mcache + if c.gcworkbuf != nil { + b = (*workbuf)(c.gcworkbuf) + c.gcworkbuf = nil + } + if b == nil { + b = (*workbuf)(lfstackpop(&work.empty)) + } + if b == nil { + b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), _CacheLineSize, &memstats.gc_sys)) + } + b.nobj = 0 + return b +} + +func putempty(b *workbuf) { + _g_ := getg() + c := _g_.m.mcache + if c.gcworkbuf == nil { + c.gcworkbuf = (unsafe.Pointer)(b) + return + } + lfstackpush(&work.empty, &b.node) +} + +func gcworkbuffree(b unsafe.Pointer) { + if b != nil { + putempty((*workbuf)(b)) + } +} + +// Get a full work buffer off the work.full list, or return nil. +func getfull(b *workbuf) *workbuf { + if b != nil { + lfstackpush(&work.empty, &b.node) + } + b = (*workbuf)(lfstackpop(&work.full)) + if b != nil || work.nproc == 1 { + return b + } + + xadd(&work.nwait, +1) + for i := 0; ; i++ { + if work.full != 0 { + xadd(&work.nwait, -1) + b = (*workbuf)(lfstackpop(&work.full)) + if b != nil { + return b + } + xadd(&work.nwait, +1) + } + if work.nwait == work.nproc { + return nil + } + _g_ := getg() + if i < 10 { + _g_.m.gcstats.nprocyield++ + procyield(20) + } else if i < 20 { + _g_.m.gcstats.nosyield++ + osyield() + } else { + _g_.m.gcstats.nsleep++ + usleep(100) + } + } +} + +func handoff(b *workbuf) *workbuf { + // Make new buffer with half of b's pointers. + b1 := getempty(nil) + n := b.nobj / 2 + b.nobj -= n + b1.nobj = n + memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), n*unsafe.Sizeof(b1.obj[0])) + _g_ := getg() + _g_.m.gcstats.nhandoff++ + _g_.m.gcstats.nhandoffcnt += uint64(n) + + // Put b on full list - let first half of b get stolen. + lfstackpush(&work.full, &b.node) + return b1 +} + +func stackmapdata(stkmap *stackmap, n int32) bitvector { + if n < 0 || n >= stkmap.n { + gothrow("stackmapdata: index out of range") + } + return bitvector{stkmap.nbit, (*byte)(add(unsafe.Pointer(&stkmap.bytedata), uintptr(n*((stkmap.nbit+31)/32*4))))} +} + +// Scan a stack frame: local variables and function arguments/results. +func scanframe(frame *stkframe, unused unsafe.Pointer) bool { + + f := frame.fn + targetpc := frame.continpc + if targetpc == 0 { + // Frame is dead. + return true + } + if _DebugGC > 1 { + print("scanframe ", gofuncname(f), "\n") + } + if targetpc != f.entry { + targetpc-- + } + pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc) + if pcdata == -1 { + // We do not have a valid pcdata value but there might be a + // stackmap for this function. It is likely that we are looking + // at the function prologue, assume so and hope for the best. + pcdata = 0 + } + + // Scan local variables if stack frame has been allocated. + size := frame.varp - frame.sp + var minsize uintptr + if thechar != '6' && thechar != '8' { + minsize = ptrSize + } else { + minsize = 0 + } + if size > minsize { + stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) + if stkmap == nil || stkmap.n <= 0 { + print("runtime: frame ", gofuncname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n") + gothrow("missing stackmap") + } + + // Locals bitmap information, scan just the pointers in locals. + if pcdata < 0 || pcdata >= stkmap.n { + // don't know where we are + print("runtime: pcdata is ", pcdata, " and ", stkmap.n, " locals stack map entries for ", gofuncname(f), " (targetpc=", targetpc, ")\n") + gothrow("scanframe: bad symbol table") + } + bv := stackmapdata(stkmap, pcdata) + size = (uintptr(bv.n) * ptrSize) / bitsPerPointer + scanblock(frame.varp-size, uintptr(bv.n)/bitsPerPointer*ptrSize, bv.bytedata) + } + + // Scan arguments. + if frame.arglen > 0 { + var bv bitvector + if frame.argmap != nil { + bv = *frame.argmap + } else { + stkmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps)) + if stkmap == nil || stkmap.n <= 0 { + print("runtime: frame ", gofuncname(f), " untyped args ", hex(frame.argp), "+", hex(frame.arglen), "\n") + gothrow("missing stackmap") + } + if pcdata < 0 || pcdata >= stkmap.n { + // don't know where we are + print("runtime: pcdata is ", pcdata, " and ", stkmap.n, " args stack map entries for ", gofuncname(f), " (targetpc=", targetpc, ")\n") + gothrow("scanframe: bad symbol table") + } + bv = stackmapdata(stkmap, pcdata) + } + scanblock(frame.argp, uintptr(bv.n)/bitsPerPointer*ptrSize, bv.bytedata) + } + return true +} + +func scanstack(gp *g) { + // TODO(rsc): Due to a precedence error, this was never checked in the original C version. + // If you enable the check, the gothrow happens. + /* + if readgstatus(gp)&_Gscan == 0 { + print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") + gothrow("mark - bad status") + } + */ + + switch readgstatus(gp) &^ _Gscan { + default: + print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") + gothrow("mark - bad status") + case _Gdead: + return + case _Grunning: + print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") + gothrow("mark - world not stopped") + case _Grunnable, _Gsyscall, _Gwaiting: + // ok + } + + if gp == getg() { + gothrow("can't scan our own stack") + } + mp := gp.m + if mp != nil && mp.helpgc != 0 { + gothrow("can't scan gchelper stack") + } + + gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, scanframe, nil, 0) + tracebackdefers(gp, scanframe, nil) +} + +// The gp has been moved to a gc safepoint. If there is gcphase specific +// work it is done here. +func gcphasework(gp *g) { + switch gcphase { + default: + gothrow("gcphasework in bad gcphase") + case _GCoff, _GCquiesce, _GCstw, _GCsweep: + // No work for now. + case _GCmark: + // Disabled until concurrent GC is implemented + // but indicate the scan has been done. + // scanstack(gp); + } + gp.gcworkdone = true +} + +var finalizer1 = [...]byte{ + // Each Finalizer is 5 words, ptr ptr uintptr ptr ptr. + // Each byte describes 4 words. + // Need 4 Finalizers described by 5 bytes before pattern repeats: + // ptr ptr uintptr ptr ptr + // ptr ptr uintptr ptr ptr + // ptr ptr uintptr ptr ptr + // ptr ptr uintptr ptr ptr + // aka + // ptr ptr uintptr ptr + // ptr ptr ptr uintptr + // ptr ptr ptr ptr + // uintptr ptr ptr ptr + // ptr uintptr ptr ptr + // Assumptions about Finalizer layout checked below. + bitsPointer | bitsPointer<<2 | bitsScalar<<4 | bitsPointer<<6, + bitsPointer | bitsPointer<<2 | bitsPointer<<4 | bitsScalar<<6, + bitsPointer | bitsPointer<<2 | bitsPointer<<4 | bitsPointer<<6, + bitsScalar | bitsPointer<<2 | bitsPointer<<4 | bitsPointer<<6, + bitsPointer | bitsScalar<<2 | bitsPointer<<4 | bitsPointer<<6, +} + +func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot *ptrtype) { + lock(&finlock) + if finq == nil || finq.cnt == finq.cap { + if finc == nil { + finc = (*finblock)(persistentalloc(_FinBlockSize, 0, &memstats.gc_sys)) + finc.cap = int32((_FinBlockSize-unsafe.Sizeof(finblock{}))/unsafe.Sizeof(finalizer{}) + 1) + finc.alllink = allfin + allfin = finc + if finptrmask[0] == 0 { + // Build pointer mask for Finalizer array in block. + // Check assumptions made in finalizer1 array above. + if (unsafe.Sizeof(finalizer{}) != 5*ptrSize || + unsafe.Offsetof(finalizer{}.fn) != 0 || + unsafe.Offsetof(finalizer{}.arg) != ptrSize || + unsafe.Offsetof(finalizer{}.nret) != 2*ptrSize || + unsafe.Offsetof(finalizer{}.fint) != 3*ptrSize || + unsafe.Offsetof(finalizer{}.ot) != 4*ptrSize || + bitsPerPointer != 2) { + gothrow("finalizer out of sync") + } + for i := range finptrmask { + finptrmask[i] = finalizer1[i%len(finalizer1)] + } + } + } + block := finc + finc = block.next + block.next = finq + finq = block + } + f := (*finalizer)(add(unsafe.Pointer(&finq.fin[0]), uintptr(finq.cnt)*unsafe.Sizeof(finq.fin[0]))) + finq.cnt++ + f.fn = fn + f.nret = nret + f.fint = fint + f.ot = ot + f.arg = p + fingwake = true + unlock(&finlock) +} + +func iterate_finq(callback func(*funcval, unsafe.Pointer, uintptr, *_type, *ptrtype)) { + for fb := allfin; fb != nil; fb = fb.alllink { + for i := int32(0); i < fb.cnt; i++ { + f := &fb.fin[i] + callback(f.fn, f.arg, f.nret, f.fint, f.ot) + } + } +} + +func mSpan_EnsureSwept(s *mspan) { + // Caller must disable preemption. + // Otherwise when this function returns the span can become unswept again + // (if GC is triggered on another goroutine). + _g_ := getg() + if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { + gothrow("MSpan_EnsureSwept: m is not locked") + } + + sg := mheap_.sweepgen + if atomicload(&s.sweepgen) == sg { + return + } + if cas(&s.sweepgen, sg-2, sg-1) { + mSpan_Sweep(s, false) + return + } + // unfortunate condition, and we don't have efficient means to wait + for atomicload(&s.sweepgen) != sg { + osyield() + } +} + +// Sweep frees or collects finalizers for blocks not marked in the mark phase. +// It clears the mark bits in preparation for the next GC round. +// Returns true if the span was returned to heap. +// If preserve=true, don't return it to heap nor relink in MCentral lists; +// caller takes care of it. +func mSpan_Sweep(s *mspan, preserve bool) bool { + // It's critical that we enter this function with preemption disabled, + // GC must not start while we are in the middle of this function. + _g_ := getg() + if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { + gothrow("MSpan_Sweep: m is not locked") + } + sweepgen := mheap_.sweepgen + if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { + print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") + gothrow("MSpan_Sweep: bad span state") + } + arena_start := mheap_.arena_start + cl := s.sizeclass + size := s.elemsize + var n int32 + var npages int32 + if cl == 0 { + n = 1 + } else { + // Chunk full of small blocks. + npages = class_to_allocnpages[cl] + n = (npages << _PageShift) / int32(size) + } + res := false + nfree := 0 + var head mlink + end := &head + c := _g_.m.mcache + sweepgenset := false + + // Mark any free objects in this span so we don't collect them. + for link := s.freelist; link != nil; link = link.next { + off := (uintptr(unsafe.Pointer(link)) - arena_start) / ptrSize + bitp := arena_start - off/wordsPerBitmapByte - 1 + shift := (off % wordsPerBitmapByte) * gcBits + *(*byte)(unsafe.Pointer(bitp)) |= bitMarked << shift + } + + // Unlink & free special records for any objects we're about to free. + specialp := &s.specials + special := *specialp + for special != nil { + // A finalizer can be set for an inner byte of an object, find object beginning. + p := uintptr(s.start<<_PageShift) + uintptr(special.offset)/size*size + off := (p - arena_start) / ptrSize + bitp := arena_start - off/wordsPerBitmapByte - 1 + shift := (off % wordsPerBitmapByte) * gcBits + bits := (*(*byte)(unsafe.Pointer(bitp)) >> shift) & bitMask + if bits&bitMarked == 0 { + // Find the exact byte for which the special was setup + // (as opposed to object beginning). + p := uintptr(s.start<<_PageShift) + uintptr(special.offset) + // about to free object: splice out special record + y := special + special = special.next + *specialp = special + if !freespecial(y, unsafe.Pointer(p), size, false) { + // stop freeing of object if it has a finalizer + *(*byte)(unsafe.Pointer(bitp)) |= bitMarked << shift + } + } else { + // object is still live: keep special record + specialp = &special.next + special = *specialp + } + } + + // Sweep through n objects of given size starting at p. + // This thread owns the span now, so it can manipulate + // the block bitmap without atomic operations. + p := uintptr(s.start << _PageShift) + off := (p - arena_start) / ptrSize + bitp := arena_start - off/wordsPerBitmapByte - 1 + shift := uint(0) + step := size / (ptrSize * wordsPerBitmapByte) + // Rewind to the previous quadruple as we move to the next + // in the beginning of the loop. + bitp += step + if step == 0 { + // 8-byte objects. + bitp++ + shift = gcBits + } + for ; n > 0; n, p = n-1, p+size { + bitp -= step + if step == 0 { + if shift != 0 { + bitp-- + } + shift = gcBits - shift + } + + xbits := *(*byte)(unsafe.Pointer(bitp)) + bits := (xbits >> shift) & bitMask + + // Allocated and marked object, reset bits to allocated. + if bits&bitMarked != 0 { + *(*byte)(unsafe.Pointer(bitp)) &^= bitMarked << shift + continue + } + + // At this point we know that we are looking at garbage object + // that needs to be collected. + if debug.allocfreetrace != 0 { + tracefree(unsafe.Pointer(p), size) + } + + // Reset to allocated+noscan. + *(*byte)(unsafe.Pointer(bitp)) = uint8(uintptr(xbits&^((bitMarked|bitsMask<<2)<<shift)) | uintptr(bitsDead)<<(shift+2)) + if cl == 0 { + // Free large span. + if preserve { + gothrow("can't preserve large span") + } + unmarkspan(p, s.npages<<_PageShift) + s.needzero = 1 + + // important to set sweepgen before returning it to heap + atomicstore(&s.sweepgen, sweepgen) + sweepgenset = true + + // NOTE(rsc,dvyukov): The original implementation of efence + // in CL 22060046 used SysFree instead of SysFault, so that + // the operating system would eventually give the memory + // back to us again, so that an efence program could run + // longer without running out of memory. Unfortunately, + // calling SysFree here without any kind of adjustment of the + // heap data structures means that when the memory does + // come back to us, we have the wrong metadata for it, either in + // the MSpan structures or in the garbage collection bitmap. + // Using SysFault here means that the program will run out of + // memory fairly quickly in efence mode, but at least it won't + // have mysterious crashes due to confused memory reuse. + // It should be possible to switch back to SysFree if we also + // implement and then call some kind of MHeap_DeleteSpan. + if debug.efence > 0 { + s.limit = 0 // prevent mlookup from finding this span + sysFault(unsafe.Pointer(p), size) + } else { + mHeap_Free(&mheap_, s, 1) + } + c.local_nlargefree++ + c.local_largefree += size + xadd64(&memstats.next_gc, -int64(size)*int64(gcpercent+100)/100) + res = true + } else { + // Free small object. + if size > 2*ptrSize { + *(*uintptr)(unsafe.Pointer(p + ptrSize)) = uintptrMask & 0xdeaddeaddeaddead // mark as "needs to be zeroed" + } else if size > ptrSize { + *(*uintptr)(unsafe.Pointer(p + ptrSize)) = 0 + } + end.next = (*mlink)(unsafe.Pointer(p)) + end = end.next + nfree++ + } + } + + // We need to set s.sweepgen = h.sweepgen only when all blocks are swept, + // because of the potential for a concurrent free/SetFinalizer. + // But we need to set it before we make the span available for allocation + // (return it to heap or mcentral), because allocation code assumes that a + // span is already swept if available for allocation. + if !sweepgenset && nfree == 0 { + // The span must be in our exclusive ownership until we update sweepgen, + // check for potential races. + if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { + print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") + gothrow("MSpan_Sweep: bad span state after sweep") + } + atomicstore(&s.sweepgen, sweepgen) + } + if nfree > 0 { + c.local_nsmallfree[cl] += uintptr(nfree) + c.local_cachealloc -= intptr(uintptr(nfree) * size) + xadd64(&memstats.next_gc, -int64(nfree)*int64(size)*int64(gcpercent+100)/100) + res = mCentral_FreeSpan(&mheap_.central[cl].mcentral, s, int32(nfree), head.next, end, preserve) + // MCentral_FreeSpan updates sweepgen + } + return res +} + +// State of background sweep. +// Protected by gclock. +type sweepdata struct { + g *g + parked bool + started bool + + spanidx uint32 // background sweeper position + + nbgsweep uint32 + npausesweep uint32 +} + +var sweep sweepdata + +// sweeps one span +// returns number of pages returned to heap, or ^uintptr(0) if there is nothing to sweep +func sweepone() uintptr { + _g_ := getg() + + // increment locks to ensure that the goroutine is not preempted + // in the middle of sweep thus leaving the span in an inconsistent state for next GC + _g_.m.locks++ + sg := mheap_.sweepgen + for { + idx := xadd(&sweep.spanidx, 1) - 1 + if idx >= uint32(len(work.spans)) { + mheap_.sweepdone = 1 + _g_.m.locks-- + return ^uintptr(0) + } + s := work.spans[idx] + if s.state != mSpanInUse { + s.sweepgen = sg + continue + } + if s.sweepgen != sg-2 || !cas(&s.sweepgen, sg-2, sg-1) { + continue + } + npages := s.npages + if !mSpan_Sweep(s, false) { + npages = 0 + } + _g_.m.locks-- + return npages + } +} + +func gosweepone() uintptr { + var ret uintptr + systemstack(func() { + ret = sweepone() + }) + return ret +} + +func gosweepdone() bool { + return mheap_.sweepdone != 0 +} + +func gchelper() { + _g_ := getg() + _g_.m.traceback = 2 + gchelperstart() + + // parallel mark for over gc roots + parfordo(work.markfor) + + // help other threads scan secondary blocks + scanblock(0, 0, nil) + + nproc := work.nproc // work.nproc can change right after we increment work.ndone + if xadd(&work.ndone, +1) == nproc-1 { + notewakeup(&work.alldone) + } + _g_.m.traceback = 0 +} + +func cachestats() { + for i := 0; ; i++ { + p := allp[i] + if p == nil { + break + } + c := p.mcache + if c == nil { + continue + } + purgecachedstats(c) + } +} + +func flushallmcaches() { + for i := 0; ; i++ { + p := allp[i] + if p == nil { + break + } + c := p.mcache + if c == nil { + continue + } + mCache_ReleaseAll(c) + stackcache_clear(c) + } +} + +func updatememstats(stats *gcstats) { + if stats != nil { + *stats = gcstats{} + } + for mp := allm; mp != nil; mp = mp.alllink { + if stats != nil { + src := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(&mp.gcstats)) + dst := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(stats)) + for i, v := range src { + dst[i] += v + } + mp.gcstats = gcstats{} + } + } + + memstats.mcache_inuse = uint64(mheap_.cachealloc.inuse) + memstats.mspan_inuse = uint64(mheap_.spanalloc.inuse) + memstats.sys = memstats.heap_sys + memstats.stacks_sys + memstats.mspan_sys + + memstats.mcache_sys + memstats.buckhash_sys + memstats.gc_sys + memstats.other_sys + + // Calculate memory allocator stats. + // During program execution we only count number of frees and amount of freed memory. + // Current number of alive object in the heap and amount of alive heap memory + // are calculated by scanning all spans. + // Total number of mallocs is calculated as number of frees plus number of alive objects. + // Similarly, total amount of allocated memory is calculated as amount of freed memory + // plus amount of alive heap memory. + memstats.alloc = 0 + memstats.total_alloc = 0 + memstats.nmalloc = 0 + memstats.nfree = 0 + for i := 0; i < len(memstats.by_size); i++ { + memstats.by_size[i].nmalloc = 0 + memstats.by_size[i].nfree = 0 + } + + // Flush MCache's to MCentral. + systemstack(flushallmcaches) + + // Aggregate local stats. + cachestats() + + // Scan all spans and count number of alive objects. + lock(&mheap_.lock) + for i := uint32(0); i < mheap_.nspan; i++ { + s := h_allspans[i] + if s.state != mSpanInUse { + continue + } + if s.sizeclass == 0 { + memstats.nmalloc++ + memstats.alloc += uint64(s.elemsize) + } else { + memstats.nmalloc += uint64(s.ref) + memstats.by_size[s.sizeclass].nmalloc += uint64(s.ref) + memstats.alloc += uint64(s.ref) * uint64(s.elemsize) + } + } + unlock(&mheap_.lock) + + // Aggregate by size class. + smallfree := uint64(0) + memstats.nfree = mheap_.nlargefree + for i := 0; i < len(memstats.by_size); i++ { + memstats.nfree += mheap_.nsmallfree[i] + memstats.by_size[i].nfree = mheap_.nsmallfree[i] + memstats.by_size[i].nmalloc += mheap_.nsmallfree[i] + smallfree += uint64(mheap_.nsmallfree[i]) * uint64(class_to_size[i]) + } + memstats.nfree += memstats.tinyallocs + memstats.nmalloc += memstats.nfree + + // Calculate derived stats. + memstats.total_alloc = uint64(memstats.alloc) + uint64(mheap_.largefree) + smallfree + memstats.heap_alloc = memstats.alloc + memstats.heap_objects = memstats.nmalloc - memstats.nfree +} + +func gcinit() { + if unsafe.Sizeof(workbuf{}) != _WorkbufSize { + gothrow("runtime: size of Workbuf is suboptimal") + } + + work.markfor = parforalloc(_MaxGcproc) + gcpercent = readgogc() + gcdatamask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcdata)), uintptr(unsafe.Pointer(&edata))-uintptr(unsafe.Pointer(&data))) + gcbssmask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcbss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss))) +} + +func gc_m(start_time int64, eagersweep bool) { + _g_ := getg() + gp := _g_.m.curg + casgstatus(gp, _Grunning, _Gwaiting) + gp.waitreason = "garbage collection" + + gc(start_time, eagersweep) + + if nbadblock > 0 { + // Work out path from root to bad block. + for { + gc(start_time, eagersweep) + if nbadblock >= int32(len(badblock)) { + gothrow("cannot find path to bad pointer") + } + } + } + + casgstatus(gp, _Gwaiting, _Grunning) +} + +func gc(start_time int64, eagersweep bool) { + if _DebugGCPtrs { + print("GC start\n") + } + + if debug.allocfreetrace > 0 { + tracegc() + } + + _g_ := getg() + _g_.m.traceback = 2 + t0 := start_time + work.tstart = start_time + + var t1 int64 + if debug.gctrace > 0 { + t1 = nanotime() + } + + // Sweep what is not sweeped by bgsweep. + for sweepone() != ^uintptr(0) { + sweep.npausesweep++ + } + + // Cache runtime.mheap_.allspans in work.spans to avoid conflicts with + // resizing/freeing allspans. + // New spans can be created while GC progresses, but they are not garbage for + // this round: + // - new stack spans can be created even while the world is stopped. + // - new malloc spans can be created during the concurrent sweep + + // Even if this is stop-the-world, a concurrent exitsyscall can allocate a stack from heap. + lock(&mheap_.lock) + // Free the old cached sweep array if necessary. + if work.spans != nil && &work.spans[0] != &h_allspans[0] { + sysFree(unsafe.Pointer(&work.spans[0]), uintptr(len(work.spans))*unsafe.Sizeof(work.spans[0]), &memstats.other_sys) + } + // Cache the current array for marking. + mheap_.gcspans = mheap_.allspans + work.spans = h_allspans + unlock(&mheap_.lock) + + work.nwait = 0 + work.ndone = 0 + work.nproc = uint32(gcprocs()) + parforsetup(work.markfor, work.nproc, uint32(_RootCount+allglen), nil, false, markroot) + if work.nproc > 1 { + noteclear(&work.alldone) + helpgc(int32(work.nproc)) + } + + var t2 int64 + if debug.gctrace > 0 { + t2 = nanotime() + } + + gchelperstart() + parfordo(work.markfor) + scanblock(0, 0, nil) + + var t3 int64 + if debug.gctrace > 0 { + t3 = nanotime() + } + + if work.nproc > 1 { + notesleep(&work.alldone) + } + + shrinkfinish() + + cachestats() + // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap + // estimate what was live heap size after previous GC (for printing only) + heap0 := memstats.next_gc * 100 / (uint64(gcpercent) + 100) + // conservatively set next_gc to high value assuming that everything is live + // concurrent/lazy sweep will reduce this number while discovering new garbage + memstats.next_gc = memstats.heap_alloc + memstats.heap_alloc*uint64(gcpercent)/100 + + t4 := nanotime() + atomicstore64(&memstats.last_gc, uint64(unixnanotime())) // must be Unix time to make sense to user + memstats.pause_ns[memstats.numgc%uint32(len(memstats.pause_ns))] = uint64(t4 - t0) + memstats.pause_end[memstats.numgc%uint32(len(memstats.pause_end))] = uint64(t4) + memstats.pause_total_ns += uint64(t4 - t0) + memstats.numgc++ + if memstats.debuggc { + print("pause ", t4-t0, "\n") + } + + if debug.gctrace > 0 { + heap1 := memstats.heap_alloc + var stats gcstats + updatememstats(&stats) + if heap1 != memstats.heap_alloc { + print("runtime: mstats skew: heap=", heap1, "/", memstats.heap_alloc, "\n") + gothrow("mstats skew") + } + obj := memstats.nmalloc - memstats.nfree + + stats.nprocyield += work.markfor.nprocyield + stats.nosyield += work.markfor.nosyield + stats.nsleep += work.markfor.nsleep + + print("gc", memstats.numgc, "(", work.nproc, "): ", + (t1-t0)/1000, "+", (t2-t1)/1000, "+", (t3-t2)/1000, "+", (t4-t3)/1000, " us, ", + heap0>>20, " -> ", heap1>>20, " MB, ", + obj, " (", memstats.nmalloc, "-", memstats.nfree, ") objects, ", + gcount(), " goroutines, ", + len(work.spans), "/", sweep.nbgsweep, "/", sweep.npausesweep, " sweeps, ", + stats.nhandoff, "(", stats.nhandoffcnt, ") handoff, ", + work.markfor.nsteal, "(", work.markfor.nstealcnt, ") steal, ", + stats.nprocyield, "/", stats.nosyield, "/", stats.nsleep, " yields\n") + sweep.nbgsweep = 0 + sweep.npausesweep = 0 + } + + // See the comment in the beginning of this function as to why we need the following. + // Even if this is still stop-the-world, a concurrent exitsyscall can allocate a stack from heap. + lock(&mheap_.lock) + // Free the old cached mark array if necessary. + if work.spans != nil && &work.spans[0] != &h_allspans[0] { + sysFree(unsafe.Pointer(&work.spans[0]), uintptr(len(work.spans))*unsafe.Sizeof(work.spans[0]), &memstats.other_sys) + } + + // Cache the current array for sweeping. + mheap_.gcspans = mheap_.allspans + mheap_.sweepgen += 2 + mheap_.sweepdone = 0 + work.spans = h_allspans + sweep.spanidx = 0 + unlock(&mheap_.lock) + + if _ConcurrentSweep && !eagersweep { + lock(&gclock) + if !sweep.started { + go bgsweep() + sweep.started = true + } else if sweep.parked { + sweep.parked = false + ready(sweep.g) + } + unlock(&gclock) + } else { + // Sweep all spans eagerly. + for sweepone() != ^uintptr(0) { + sweep.npausesweep++ + } + // Do an additional mProf_GC, because all 'free' events are now real as well. + mProf_GC() + } + + mProf_GC() + _g_.m.traceback = 0 + + if _DebugGCPtrs { + print("GC end\n") + } +} + +func readmemstats_m(stats *MemStats) { + updatememstats(nil) + + // Size of the trailing by_size array differs between Go and C, + // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. + memmove(unsafe.Pointer(stats), unsafe.Pointer(&memstats), sizeof_C_MStats) + + // Stack numbers are part of the heap numbers, separate those out for user consumption + stats.StackSys = stats.StackInuse + stats.HeapInuse -= stats.StackInuse + stats.HeapSys -= stats.StackInuse +} + +//go:linkname readGCStats runtime/debug.readGCStats +func readGCStats(pauses *[]uint64) { + systemstack(func() { + readGCStats_m(pauses) + }) +} + +func readGCStats_m(pauses *[]uint64) { + p := *pauses + // Calling code in runtime/debug should make the slice large enough. + if cap(p) < len(memstats.pause_ns)+3 { + gothrow("runtime: short slice passed to readGCStats") + } + + // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns. + lock(&mheap_.lock) + + n := memstats.numgc + if n > uint32(len(memstats.pause_ns)) { + n = uint32(len(memstats.pause_ns)) + } + + // The pause buffer is circular. The most recent pause is at + // pause_ns[(numgc-1)%len(pause_ns)], and then backward + // from there to go back farther in time. We deliver the times + // most recent first (in p[0]). + p = p[:cap(p)] + for i := uint32(0); i < n; i++ { + j := (memstats.numgc - 1 - i) % uint32(len(memstats.pause_ns)) + p[i] = memstats.pause_ns[j] + p[n+i] = memstats.pause_end[j] + } + + p[n+n] = memstats.last_gc + p[n+n+1] = uint64(memstats.numgc) + p[n+n+2] = memstats.pause_total_ns + unlock(&mheap_.lock) + *pauses = p[:n+n+3] +} + +func setGCPercent(in int32) (out int32) { + lock(&mheap_.lock) + out = gcpercent + if in < 0 { + in = -1 + } + gcpercent = in + unlock(&mheap_.lock) + return out +} + +func gchelperstart() { + _g_ := getg() + + if _g_.m.helpgc < 0 || _g_.m.helpgc >= _MaxGcproc { + gothrow("gchelperstart: bad m->helpgc") + } + if _g_ != _g_.m.g0 { + gothrow("gchelper not running on g0 stack") + } +} + +func wakefing() *g { + var res *g + lock(&finlock) + if fingwait && fingwake { + fingwait = false + fingwake = false + res = fing + } + unlock(&finlock) + return res +} + +func addb(p *byte, n uintptr) *byte { + return (*byte)(add(unsafe.Pointer(p), n)) +} + +// Recursively unrolls GC program in prog. +// mask is where to store the result. +// ppos is a pointer to position in mask, in bits. +// sparse says to generate 4-bits per word mask for heap (2-bits for data/bss otherwise). +func unrollgcprog1(maskp *byte, prog *byte, ppos *uintptr, inplace, sparse bool) *byte { + arena_start := mheap_.arena_start + pos := *ppos + mask := (*[1 << 30]byte)(unsafe.Pointer(maskp)) + for { + switch *prog { + default: + gothrow("unrollgcprog: unknown instruction") + + case insData: + prog = addb(prog, 1) + siz := int(*prog) + prog = addb(prog, 1) + p := (*[1 << 30]byte)(unsafe.Pointer(prog)) + for i := 0; i < siz; i++ { + v := p[i/_PointersPerByte] + v >>= (uint(i) % _PointersPerByte) * _BitsPerPointer + v &= _BitsMask + if inplace { + // Store directly into GC bitmap. + off := (uintptr(unsafe.Pointer(&mask[pos])) - arena_start) / ptrSize + bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)) + shift := (off % wordsPerBitmapByte) * gcBits + if shift == 0 { + *bitp = 0 + } + *bitp |= v << (shift + 2) + pos += ptrSize + } else if sparse { + // 4-bits per word + v <<= (pos % 8) + 2 + mask[pos/8] |= v + pos += gcBits + } else { + // 2-bits per word + v <<= pos % 8 + mask[pos/8] |= v + pos += _BitsPerPointer + } + } + prog = addb(prog, round(uintptr(siz)*_BitsPerPointer, 8)/8) + + case insArray: + prog = (*byte)(add(unsafe.Pointer(prog), 1)) + siz := uintptr(0) + for i := uintptr(0); i < ptrSize; i++ { + siz = (siz << 8) + uintptr(*(*byte)(add(unsafe.Pointer(prog), ptrSize-i-1))) + } + prog = (*byte)(add(unsafe.Pointer(prog), ptrSize)) + var prog1 *byte + for i := uintptr(0); i < siz; i++ { + prog1 = unrollgcprog1(&mask[0], prog, &pos, inplace, sparse) + } + if *prog1 != insArrayEnd { + gothrow("unrollgcprog: array does not end with insArrayEnd") + } + prog = (*byte)(add(unsafe.Pointer(prog1), 1)) + + case insArrayEnd, insEnd: + *ppos = pos + return prog + } + } +} + +// Unrolls GC program prog for data/bss, returns dense GC mask. +func unrollglobgcprog(prog *byte, size uintptr) bitvector { + masksize := round(round(size, ptrSize)/ptrSize*bitsPerPointer, 8) / 8 + mask := (*[1 << 30]byte)(persistentalloc(masksize+1, 0, &memstats.gc_sys)) + mask[masksize] = 0xa1 + pos := uintptr(0) + prog = unrollgcprog1(&mask[0], prog, &pos, false, false) + if pos != size/ptrSize*bitsPerPointer { + print("unrollglobgcprog: bad program size, got ", pos, ", expect ", size/ptrSize*bitsPerPointer, "\n") + gothrow("unrollglobgcprog: bad program size") + } + if *prog != insEnd { + gothrow("unrollglobgcprog: program does not end with insEnd") + } + if mask[masksize] != 0xa1 { + gothrow("unrollglobgcprog: overflow") + } + return bitvector{int32(masksize * 8), &mask[0]} +} + +func unrollgcproginplace_m(v unsafe.Pointer, typ *_type, size, size0 uintptr) { + pos := uintptr(0) + prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1]))) + for pos != size0 { + unrollgcprog1((*byte)(v), prog, &pos, true, true) + } + + // Mark first word as bitAllocated. + arena_start := mheap_.arena_start + off := (uintptr(v) - arena_start) / ptrSize + bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)) + shift := (off % wordsPerBitmapByte) * gcBits + *bitp |= bitBoundary << shift + + // Mark word after last as BitsDead. + if size0 < size { + off := (uintptr(v) + size0 - arena_start) / ptrSize + bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)) + shift := (off % wordsPerBitmapByte) * gcBits + *bitp &= uint8(^(bitPtrMask << shift) | uintptr(bitsDead)<<(shift+2)) + } +} + +var unroll mutex + +// Unrolls GC program in typ.gc[1] into typ.gc[0] +func unrollgcprog_m(typ *_type) { + lock(&unroll) + mask := (*byte)(unsafe.Pointer(uintptr(typ.gc[0]))) + if *mask == 0 { + pos := uintptr(8) // skip the unroll flag + prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1]))) + prog = unrollgcprog1(mask, prog, &pos, false, true) + if *prog != insEnd { + gothrow("unrollgcprog: program does not end with insEnd") + } + if typ.size/ptrSize%2 != 0 { + // repeat the program + prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1]))) + unrollgcprog1(mask, prog, &pos, false, true) + } + + // atomic way to say mask[0] = 1 + atomicor8(mask, 1) + } + unlock(&unroll) +} + +// mark the span of memory at v as having n blocks of the given size. +// if leftover is true, there is left over space at the end of the span. +func markspan(v unsafe.Pointer, size uintptr, n uintptr, leftover bool) { + if uintptr(v)+size*n > mheap_.arena_used || uintptr(v) < mheap_.arena_start { + gothrow("markspan: bad pointer") + } + + // Find bits of the beginning of the span. + off := (uintptr(v) - uintptr(mheap_.arena_start)) / ptrSize + if off%wordsPerBitmapByte != 0 { + gothrow("markspan: unaligned length") + } + b := mheap_.arena_start - off/wordsPerBitmapByte - 1 + + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap byte has bits for only + // one span, so no other goroutines are changing these bitmap words. + + if size == ptrSize { + // Possible only on 64-bits (minimal size class is 8 bytes). + // Set memory to 0x11. + if (bitBoundary|bitsDead)<<gcBits|bitBoundary|bitsDead != 0x11 { + gothrow("markspan: bad bits") + } + if n%(wordsPerBitmapByte*ptrSize) != 0 { + gothrow("markspan: unaligned length") + } + b = b - n/wordsPerBitmapByte + 1 // find first byte + if b%ptrSize != 0 { + gothrow("markspan: unaligned pointer") + } + for i := uintptr(0); i < n; i, b = i+wordsPerBitmapByte*ptrSize, b+ptrSize { + *(*uintptr)(unsafe.Pointer(b)) = uintptrMask & 0x1111111111111111 // bitBoundary | bitsDead, repeated + } + return + } + + if leftover { + n++ // mark a boundary just past end of last block too + } + step := size / (ptrSize * wordsPerBitmapByte) + for i := uintptr(0); i < n; i, b = i+1, b-step { + *(*byte)(unsafe.Pointer(b)) = bitBoundary | bitsDead<<2 + } +} + +// unmark the span of memory at v of length n bytes. +func unmarkspan(v, n uintptr) { + if v+n > mheap_.arena_used || v < mheap_.arena_start { + gothrow("markspan: bad pointer") + } + + off := (v - mheap_.arena_start) / ptrSize // word offset + if off%(ptrSize*wordsPerBitmapByte) != 0 { + gothrow("markspan: unaligned pointer") + } + + b := mheap_.arena_start - off/wordsPerBitmapByte - 1 + n /= ptrSize + if n%(ptrSize*wordsPerBitmapByte) != 0 { + gothrow("unmarkspan: unaligned length") + } + + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap word has bits for only + // one span, so no other goroutines are changing these + // bitmap words. + n /= wordsPerBitmapByte + memclr(unsafe.Pointer(b-n+1), n) +} + +func mHeap_MapBits(h *mheap) { + // Caller has added extra mappings to the arena. + // Add extra mappings of bitmap words as needed. + // We allocate extra bitmap pieces in chunks of bitmapChunk. + const bitmapChunk = 8192 + + n := (h.arena_used - h.arena_start) / (ptrSize * wordsPerBitmapByte) + n = round(n, bitmapChunk) + n = round(n, _PhysPageSize) + if h.bitmap_mapped >= n { + return + } + + sysMap(unsafe.Pointer(h.arena_start-n), n-h.bitmap_mapped, h.arena_reserved, &memstats.gc_sys) + h.bitmap_mapped = n +} + +func getgcmaskcb(frame *stkframe, ctxt unsafe.Pointer) bool { + target := (*stkframe)(ctxt) + if frame.sp <= target.sp && target.sp < frame.varp { + *target = *frame + return false + } + return true +} + +// Returns GC type info for object p for testing. +func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) { + *mask = nil + *len = 0 + + // data + if uintptr(unsafe.Pointer(&data)) <= uintptr(p) && uintptr(p) < uintptr(unsafe.Pointer(&edata)) { + n := (*ptrtype)(unsafe.Pointer(t)).elem.size + *len = n / ptrSize + *mask = &make([]byte, *len)[0] + for i := uintptr(0); i < n; i += ptrSize { + off := (uintptr(p) + i - uintptr(unsafe.Pointer(&data))) / ptrSize + bits := (*(*byte)(add(unsafe.Pointer(gcdatamask.bytedata), off/pointersPerByte)) >> ((off % pointersPerByte) * bitsPerPointer)) & bitsMask + *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits + } + return + } + + // bss + if uintptr(unsafe.Pointer(&bss)) <= uintptr(p) && uintptr(p) < uintptr(unsafe.Pointer(&ebss)) { + n := (*ptrtype)(unsafe.Pointer(t)).elem.size + *len = n / ptrSize + *mask = &make([]byte, *len)[0] + for i := uintptr(0); i < n; i += ptrSize { + off := (uintptr(p) + i - uintptr(unsafe.Pointer(&bss))) / ptrSize + bits := (*(*byte)(add(unsafe.Pointer(gcbssmask.bytedata), off/pointersPerByte)) >> ((off % pointersPerByte) * bitsPerPointer)) & bitsMask + *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits + } + return + } + + // heap + var n uintptr + var base uintptr + if mlookup(uintptr(p), &base, &n, nil) != 0 { + *len = n / ptrSize + *mask = &make([]byte, *len)[0] + for i := uintptr(0); i < n; i += ptrSize { + off := (uintptr(base) + i - mheap_.arena_start) / ptrSize + b := mheap_.arena_start - off/wordsPerBitmapByte - 1 + shift := (off % wordsPerBitmapByte) * gcBits + bits := (*(*byte)(unsafe.Pointer(b)) >> (shift + 2)) & bitsMask + *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits + } + return + } + + // stack + var frame stkframe + frame.sp = uintptr(p) + _g_ := getg() + gentraceback(_g_.m.curg.sched.pc, _g_.m.curg.sched.sp, 0, _g_.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0) + if frame.fn != nil { + f := frame.fn + targetpc := frame.continpc + if targetpc == 0 { + return + } + if targetpc != f.entry { + targetpc-- + } + pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc) + if pcdata == -1 { + return + } + stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) + if stkmap == nil || stkmap.n <= 0 { + return + } + bv := stackmapdata(stkmap, pcdata) + size := uintptr(bv.n) / bitsPerPointer * ptrSize + n := (*ptrtype)(unsafe.Pointer(t)).elem.size + *len = n / ptrSize + *mask = &make([]byte, *len)[0] + for i := uintptr(0); i < n; i += ptrSize { + off := (uintptr(p) + i - frame.varp + size) / ptrSize + bits := ((*(*byte)(add(unsafe.Pointer(bv.bytedata), off*bitsPerPointer/8))) >> ((off * bitsPerPointer) % 8)) & bitsMask + *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits + } + } +} + +func unixnanotime() int64 { + var now int64 + gc_unixnanotime(&now) + return now +} diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c deleted file mode 100644 index 7754bad89..000000000 --- a/src/runtime/mgc0.c +++ /dev/null @@ -1,2010 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Garbage collector (GC). -// -// GC is: -// - mark&sweep -// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc) -// - parallel (up to MaxGcproc threads) -// - partially concurrent (mark is stop-the-world, while sweep is concurrent) -// - non-moving/non-compacting -// - full (non-partial) -// -// GC rate. -// Next GC is after we've allocated an extra amount of memory proportional to -// the amount already in use. The proportion is controlled by GOGC environment variable -// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M -// (this mark is tracked in next_gc variable). This keeps the GC cost in linear -// proportion to the allocation cost. Adjusting GOGC just changes the linear constant -// (and also the amount of extra memory used). -// -// Concurrent sweep. -// The sweep phase proceeds concurrently with normal program execution. -// The heap is swept span-by-span both lazily (when a goroutine needs another span) -// and concurrently in a background goroutine (this helps programs that are not CPU bound). -// However, at the end of the stop-the-world GC phase we don't know the size of the live heap, -// and so next_gc calculation is tricky and happens as follows. -// At the end of the stop-the-world phase next_gc is conservatively set based on total -// heap size; all spans are marked as "needs sweeping". -// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory. -// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc -// closer to the target value. However, this is not enough to avoid over-allocating memory. -// Consider that a goroutine wants to allocate a new span for a large object and -// there are no free swept spans, but there are small-object unswept spans. -// If the goroutine naively allocates a new span, it can surpass the yet-unknown -// target next_gc value. In order to prevent such cases (1) when a goroutine needs -// to allocate a new small-object span, it sweeps small-object spans for the same -// object size until it frees at least one object; (2) when a goroutine needs to -// allocate large-object span from heap, it sweeps spans until it frees at least -// that many pages into heap. Together these two measures ensure that we don't surpass -// target next_gc value by a large margin. There is an exception: if a goroutine sweeps -// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span, -// but there can still be other one-page unswept spans which could be combined into a two-page span. -// It's critical to ensure that no operations proceed on unswept spans (that would corrupt -// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache, -// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it. -// When a goroutine explicitly frees an object or sets a finalizer, it ensures that -// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish). -// The finalizer goroutine is kicked off only when all spans are swept. -// When the next GC starts, it sweeps all not-yet-swept spans (if any). - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "stack.h" -#include "mgc0.h" -#include "chan.h" -#include "race.h" -#include "type.h" -#include "typekind.h" -#include "funcdata.h" -#include "textflag.h" - -enum { - Debug = 0, - DebugPtrs = 0, // if 1, print trace of every pointer load during GC - ConcurrentSweep = 1, - - WorkbufSize = 4*1024, - FinBlockSize = 4*1024, - RootData = 0, - RootBss = 1, - RootFinalizers = 2, - RootSpans = 3, - RootFlushCaches = 4, - RootCount = 5, -}; - -// ptrmask for an allocation containing a single pointer. -static byte oneptr[] = {BitsPointer}; - -// Initialized from $GOGC. GOGC=off means no gc. -extern int32 runtime·gcpercent; - -// Holding worldsema grants an M the right to try to stop the world. -// The procedure is: -// -// runtime·semacquire(&runtime·worldsema); -// m->gcing = 1; -// runtime·stoptheworld(); -// -// ... do stuff ... -// -// m->gcing = 0; -// runtime·semrelease(&runtime·worldsema); -// runtime·starttheworld(); -// -uint32 runtime·worldsema = 1; - -typedef struct Workbuf Workbuf; -struct Workbuf -{ - LFNode node; // must be first - uintptr nobj; - byte* obj[(WorkbufSize-sizeof(LFNode)-sizeof(uintptr))/PtrSize]; -}; - -extern byte runtime·data[]; -extern byte runtime·edata[]; -extern byte runtime·bss[]; -extern byte runtime·ebss[]; - -extern byte runtime·gcdata[]; -extern byte runtime·gcbss[]; - -Mutex runtime·finlock; // protects the following variables -G* runtime·fing; // goroutine that runs finalizers -FinBlock* runtime·finq; // list of finalizers that are to be executed -FinBlock* runtime·finc; // cache of free blocks -static byte finptrmask[FinBlockSize/PtrSize/PointersPerByte]; -bool runtime·fingwait; -bool runtime·fingwake; -FinBlock *runtime·allfin; // list of all blocks - -BitVector runtime·gcdatamask; -BitVector runtime·gcbssmask; - -Mutex runtime·gclock; - -static uintptr badblock[1024]; -static int32 nbadblock; - -static Workbuf* getempty(Workbuf*); -static Workbuf* getfull(Workbuf*); -static void putempty(Workbuf*); -static Workbuf* handoff(Workbuf*); -static void gchelperstart(void); -static void flushallmcaches(void); -static bool scanframe(Stkframe *frame, void *unused); -static void scanstack(G *gp); -static BitVector unrollglobgcprog(byte *prog, uintptr size); - -void runtime·bgsweep(void); -static FuncVal bgsweepv = {runtime·bgsweep}; - -typedef struct WorkData WorkData; -struct WorkData { - uint64 full; // lock-free list of full blocks - uint64 empty; // lock-free list of empty blocks - byte pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait - uint32 nproc; - int64 tstart; - volatile uint32 nwait; - volatile uint32 ndone; - Note alldone; - ParFor* markfor; - - // Copy of mheap.allspans for marker or sweeper. - MSpan** spans; - uint32 nspan; -}; -WorkData runtime·work; - -// Is _cgo_allocate linked into the binary? -static bool -have_cgo_allocate(void) -{ - extern byte go·weak·runtime·_cgo_allocate_internal[1]; - return go·weak·runtime·_cgo_allocate_internal != nil; -} - -// scanblock scans a block of n bytes starting at pointer b for references -// to other objects, scanning any it finds recursively until there are no -// unscanned objects left. Instead of using an explicit recursion, it keeps -// a work list in the Workbuf* structures and loops in the main function -// body. Keeping an explicit work list is easier on the stack allocator and -// more efficient. -static void -scanblock(byte *b, uintptr n, byte *ptrmask) -{ - byte *obj, *obj0, *p, *arena_start, *arena_used, **wp, *scanbuf[8], *ptrbitp, *bitp; - uintptr i, j, nobj, size, idx, x, off, scanbufpos, bits, xbits, shift; - Workbuf *wbuf; - Iface *iface; - Eface *eface; - Type *typ; - MSpan *s; - pageID k; - bool keepworking; - - // Cache memory arena parameters in local vars. - arena_start = runtime·mheap.arena_start; - arena_used = runtime·mheap.arena_used; - - wbuf = getempty(nil); - nobj = wbuf->nobj; - wp = &wbuf->obj[nobj]; - keepworking = b == nil; - scanbufpos = 0; - for(i = 0; i < nelem(scanbuf); i++) - scanbuf[i] = nil; - - ptrbitp = nil; - - // ptrmask can have 2 possible values: - // 1. nil - obtain pointer mask from GC bitmap. - // 2. pointer to a compact mask (for stacks and data). - if(b != nil) - goto scanobj; - for(;;) { - if(nobj == 0) { - // Out of work in workbuf. - // First, see is there is any work in scanbuf. - for(i = 0; i < nelem(scanbuf); i++) { - b = scanbuf[scanbufpos]; - scanbuf[scanbufpos++] = nil; - scanbufpos %= nelem(scanbuf); - if(b != nil) { - n = arena_used - b; // scan until bitBoundary or BitsDead - ptrmask = nil; // use GC bitmap for pointer info - goto scanobj; - } - } - if(!keepworking) { - putempty(wbuf); - return; - } - // Refill workbuf from global queue. - wbuf = getfull(wbuf); - if(wbuf == nil) - return; - nobj = wbuf->nobj; - wp = &wbuf->obj[nobj]; - } - - // If another proc wants a pointer, give it some. - if(runtime·work.nwait > 0 && nobj > 4 && runtime·work.full == 0) { - wbuf->nobj = nobj; - wbuf = handoff(wbuf); - nobj = wbuf->nobj; - wp = &wbuf->obj[nobj]; - } - - wp--; - nobj--; - b = *wp; - n = arena_used - b; // scan until next bitBoundary or BitsDead - ptrmask = nil; // use GC bitmap for pointer info - - scanobj: - if(DebugPtrs) - runtime·printf("scanblock %p +%p %p\n", b, n, ptrmask); - // Find bits of the beginning of the object. - if(ptrmask == nil) { - off = (uintptr*)b - (uintptr*)arena_start; - ptrbitp = arena_start - off/wordsPerBitmapByte - 1; - } - for(i = 0; i < n; i += PtrSize) { - obj = nil; - // Find bits for this word. - if(ptrmask == nil) { - // Check is we have reached end of span. - if((((uintptr)b+i)%PageSize) == 0 && - runtime·mheap.spans[(b-arena_start)>>PageShift] != runtime·mheap.spans[(b+i-arena_start)>>PageShift]) - break; - // Consult GC bitmap. - bits = *ptrbitp; - - if(wordsPerBitmapByte != 2) - runtime·throw("alg doesn't work for wordsPerBitmapByte != 2"); - j = ((uintptr)b+i)/PtrSize & 1; - ptrbitp -= j; - bits >>= gcBits*j; - - if((bits&bitBoundary) != 0 && i != 0) - break; // reached beginning of the next object - bits = (bits>>2)&BitsMask; - if(bits == BitsDead) - break; // reached no-scan part of the object - } else // dense mask (stack or data) - bits = (ptrmask[(i/PtrSize)/4]>>(((i/PtrSize)%4)*BitsPerPointer))&BitsMask; - - if(bits <= BitsScalar) // BitsScalar || BitsDead - continue; - if(bits == BitsPointer) { - obj = *(byte**)(b+i); - obj0 = obj; - goto markobj; - } - - // With those three out of the way, must be multi-word. - if(Debug && bits != BitsMultiWord) - runtime·throw("unexpected garbage collection bits"); - // Find the next pair of bits. - if(ptrmask == nil) { - bits = *ptrbitp; - j = ((uintptr)b+i+PtrSize)/PtrSize & 1; - ptrbitp -= j; - bits >>= gcBits*j; - bits = (bits>>2)&BitsMask; - } else - bits = (ptrmask[((i+PtrSize)/PtrSize)/4]>>((((i+PtrSize)/PtrSize)%4)*BitsPerPointer))&BitsMask; - - if(Debug && bits != BitsIface && bits != BitsEface) - runtime·throw("unexpected garbage collection bits"); - - if(bits == BitsIface) { - iface = (Iface*)(b+i); - if(iface->tab != nil) { - typ = iface->tab->type; - if(!(typ->kind&KindDirectIface) || !(typ->kind&KindNoPointers)) - obj = iface->data; - } - } else { - eface = (Eface*)(b+i); - typ = eface->type; - if(typ != nil) { - if(!(typ->kind&KindDirectIface) || !(typ->kind&KindNoPointers)) - obj = eface->data; - } - } - - i += PtrSize; - - obj0 = obj; - markobj: - // At this point we have extracted the next potential pointer. - // Check if it points into heap. - if(obj == nil) - continue; - if(obj < arena_start || obj >= arena_used) { - if((uintptr)obj < PhysPageSize && runtime·invalidptr) { - s = nil; - goto badobj; - } - continue; - } - // Mark the object. - obj = (byte*)((uintptr)obj & ~(PtrSize-1)); - off = (uintptr*)obj - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - xbits = *bitp; - bits = (xbits >> shift) & bitMask; - if((bits&bitBoundary) == 0) { - // Not a beginning of a block, consult span table to find the block beginning. - k = (uintptr)obj>>PageShift; - x = k; - x -= (uintptr)arena_start>>PageShift; - s = runtime·mheap.spans[x]; - if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse) { - // Stack pointers lie within the arena bounds but are not part of the GC heap. - // Ignore them. - if(s != nil && s->state == MSpanStack) - continue; - - badobj: - // If cgo_allocate is linked into the binary, it can allocate - // memory as []unsafe.Pointer that may not contain actual - // pointers and must be scanned conservatively. - // In this case alone, allow the bad pointer. - if(have_cgo_allocate() && ptrmask == nil) - continue; - - // Anything else indicates a bug somewhere. - // If we're in the middle of chasing down a different bad pointer, - // don't confuse the trace by printing about this one. - if(nbadblock > 0) - continue; - - runtime·printf("runtime: garbage collector found invalid heap pointer *(%p+%p)=%p", b, i, obj); - if(s == nil) - runtime·printf(" s=nil\n"); - else - runtime·printf(" span=%p-%p-%p state=%d\n", (uintptr)s->start<<PageShift, s->limit, (uintptr)(s->start+s->npages)<<PageShift, s->state); - if(ptrmask != nil) - runtime·throw("invalid heap pointer"); - // Add to badblock list, which will cause the garbage collection - // to keep repeating until it has traced the chain of pointers - // leading to obj all the way back to a root. - if(nbadblock == 0) - badblock[nbadblock++] = (uintptr)b; - continue; - } - p = (byte*)((uintptr)s->start<<PageShift); - if(s->sizeclass != 0) { - size = s->elemsize; - idx = ((byte*)obj - p)/size; - p = p+idx*size; - } - if(p == obj) { - runtime·printf("runtime: failed to find block beginning for %p s=%p s->limit=%p\n", - p, s->start*PageSize, s->limit); - runtime·throw("failed to find block beginning"); - } - obj = p; - goto markobj; - } - if(DebugPtrs) - runtime·printf("scan *%p = %p => base %p\n", b+i, obj0, obj); - - if(nbadblock > 0 && (uintptr)obj == badblock[nbadblock-1]) { - // Running garbage collection again because - // we want to find the path from a root to a bad pointer. - // Found possible next step; extend or finish path. - for(j=0; j<nbadblock; j++) - if(badblock[j] == (uintptr)b) - goto AlreadyBad; - runtime·printf("runtime: found *(%p+%p) = %p+%p\n", b, i, obj0, (uintptr)(obj-obj0)); - if(ptrmask != nil) - runtime·throw("bad pointer"); - if(nbadblock >= nelem(badblock)) - runtime·throw("badblock trace too long"); - badblock[nbadblock++] = (uintptr)b; - AlreadyBad:; - } - - // Now we have bits, bitp, and shift correct for - // obj pointing at the base of the object. - // Only care about not marked objects. - if((bits&bitMarked) != 0) - continue; - // If obj size is greater than 8, then each byte of GC bitmap - // contains info for at most one object. In such case we use - // non-atomic byte store to mark the object. This can lead - // to double enqueue of the object for scanning, but scanning - // is an idempotent operation, so it is OK. This cannot lead - // to bitmap corruption because the single marked bit is the - // only thing that can change in the byte. - // For 8-byte objects we use non-atomic store, if the other - // quadruple is already marked. Otherwise we resort to CAS - // loop for marking. - if((xbits&(bitMask|(bitMask<<gcBits))) != (bitBoundary|(bitBoundary<<gcBits)) || - runtime·work.nproc == 1) - *bitp = xbits | (bitMarked<<shift); - else - runtime·atomicor8(bitp, bitMarked<<shift); - - if(((xbits>>(shift+2))&BitsMask) == BitsDead) - continue; // noscan object - - // Queue the obj for scanning. - PREFETCH(obj); - p = scanbuf[scanbufpos]; - scanbuf[scanbufpos++] = obj; - scanbufpos %= nelem(scanbuf); - if(p == nil) - continue; - - // If workbuf is full, obtain an empty one. - if(nobj >= nelem(wbuf->obj)) { - wbuf->nobj = nobj; - wbuf = getempty(wbuf); - nobj = wbuf->nobj; - wp = &wbuf->obj[nobj]; - } - *wp = p; - wp++; - nobj++; - } - if(DebugPtrs) - runtime·printf("end scanblock %p +%p %p\n", b, n, ptrmask); - - if(Debug && ptrmask == nil) { - // For heap objects ensure that we did not overscan. - n = 0; - p = nil; - if(!runtime·mlookup(b, &p, &n, nil) || b != p || i > n) { - runtime·printf("runtime: scanned (%p,%p), heap object (%p,%p)\n", b, i, p, n); - runtime·throw("scanblock: scanned invalid object"); - } - } - } -} - -static void -markroot(ParFor *desc, uint32 i) -{ - FinBlock *fb; - MSpan *s; - uint32 spanidx, sg; - G *gp; - void *p; - uint32 status; - bool restart; - - USED(&desc); - // Note: if you add a case here, please also update heapdump.c:dumproots. - switch(i) { - case RootData: - scanblock(runtime·data, runtime·edata - runtime·data, runtime·gcdatamask.bytedata); - break; - - case RootBss: - scanblock(runtime·bss, runtime·ebss - runtime·bss, runtime·gcbssmask.bytedata); - break; - - case RootFinalizers: - for(fb=runtime·allfin; fb; fb=fb->alllink) - scanblock((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), finptrmask); - break; - - case RootSpans: - // mark MSpan.specials - sg = runtime·mheap.sweepgen; - for(spanidx=0; spanidx<runtime·work.nspan; spanidx++) { - Special *sp; - SpecialFinalizer *spf; - - s = runtime·work.spans[spanidx]; - if(s->state != MSpanInUse) - continue; - if(s->sweepgen != sg) { - runtime·printf("sweep %d %d\n", s->sweepgen, sg); - runtime·throw("gc: unswept span"); - } - for(sp = s->specials; sp != nil; sp = sp->next) { - if(sp->kind != KindSpecialFinalizer) - continue; - // don't mark finalized object, but scan it so we - // retain everything it points to. - spf = (SpecialFinalizer*)sp; - // A finalizer can be set for an inner byte of an object, find object beginning. - p = (void*)((s->start << PageShift) + spf->special.offset/s->elemsize*s->elemsize); - scanblock(p, s->elemsize, nil); - scanblock((void*)&spf->fn, PtrSize, oneptr); - } - } - break; - - case RootFlushCaches: - flushallmcaches(); - break; - - default: - // the rest is scanning goroutine stacks - if(i - RootCount >= runtime·allglen) - runtime·throw("markroot: bad index"); - gp = runtime·allg[i - RootCount]; - // remember when we've first observed the G blocked - // needed only to output in traceback - status = runtime·readgstatus(gp); - if((status == Gwaiting || status == Gsyscall) && gp->waitsince == 0) - gp->waitsince = runtime·work.tstart; - // Shrink a stack if not much of it is being used. - runtime·shrinkstack(gp); - if(runtime·readgstatus(gp) == Gdead) - gp->gcworkdone = true; - else - gp->gcworkdone = false; - restart = runtime·stopg(gp); - scanstack(gp); - if(restart) - runtime·restartg(gp); - break; - } -} - -// Get an empty work buffer off the work.empty list, -// allocating new buffers as needed. -static Workbuf* -getempty(Workbuf *b) -{ - MCache *c; - - if(b != nil) - runtime·lfstackpush(&runtime·work.full, &b->node); - b = nil; - c = g->m->mcache; - if(c->gcworkbuf != nil) { - b = c->gcworkbuf; - c->gcworkbuf = nil; - } - if(b == nil) - b = (Workbuf*)runtime·lfstackpop(&runtime·work.empty); - if(b == nil) - b = runtime·persistentalloc(sizeof(*b), CacheLineSize, &mstats.gc_sys); - b->nobj = 0; - return b; -} - -static void -putempty(Workbuf *b) -{ - MCache *c; - - c = g->m->mcache; - if(c->gcworkbuf == nil) { - c->gcworkbuf = b; - return; - } - runtime·lfstackpush(&runtime·work.empty, &b->node); -} - -void -runtime·gcworkbuffree(void *b) -{ - if(b != nil) - putempty(b); -} - -// Get a full work buffer off the work.full list, or return nil. -static Workbuf* -getfull(Workbuf *b) -{ - int32 i; - - if(b != nil) - runtime·lfstackpush(&runtime·work.empty, &b->node); - b = (Workbuf*)runtime·lfstackpop(&runtime·work.full); - if(b != nil || runtime·work.nproc == 1) - return b; - - runtime·xadd(&runtime·work.nwait, +1); - for(i=0;; i++) { - if(runtime·work.full != 0) { - runtime·xadd(&runtime·work.nwait, -1); - b = (Workbuf*)runtime·lfstackpop(&runtime·work.full); - if(b != nil) - return b; - runtime·xadd(&runtime·work.nwait, +1); - } - if(runtime·work.nwait == runtime·work.nproc) - return nil; - if(i < 10) { - g->m->gcstats.nprocyield++; - runtime·procyield(20); - } else if(i < 20) { - g->m->gcstats.nosyield++; - runtime·osyield(); - } else { - g->m->gcstats.nsleep++; - runtime·usleep(100); - } - } -} - -static Workbuf* -handoff(Workbuf *b) -{ - int32 n; - Workbuf *b1; - - // Make new buffer with half of b's pointers. - b1 = getempty(nil); - n = b->nobj/2; - b->nobj -= n; - b1->nobj = n; - runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]); - g->m->gcstats.nhandoff++; - g->m->gcstats.nhandoffcnt += n; - - // Put b on full list - let first half of b get stolen. - runtime·lfstackpush(&runtime·work.full, &b->node); - return b1; -} - -BitVector -runtime·stackmapdata(StackMap *stackmap, int32 n) -{ - if(n < 0 || n >= stackmap->n) - runtime·throw("stackmapdata: index out of range"); - return (BitVector){stackmap->nbit, stackmap->bytedata + n*((stackmap->nbit+31)/32*4)}; -} - -// Scan a stack frame: local variables and function arguments/results. -static bool -scanframe(Stkframe *frame, void *unused) -{ - Func *f; - StackMap *stackmap; - BitVector bv; - uintptr size, minsize; - uintptr targetpc; - int32 pcdata; - - USED(unused); - f = frame->fn; - targetpc = frame->continpc; - if(targetpc == 0) { - // Frame is dead. - return true; - } - if(Debug > 1) - runtime·printf("scanframe %s\n", runtime·funcname(f)); - if(targetpc != f->entry) - targetpc--; - pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); - if(pcdata == -1) { - // We do not have a valid pcdata value but there might be a - // stackmap for this function. It is likely that we are looking - // at the function prologue, assume so and hope for the best. - pcdata = 0; - } - - // Scan local variables if stack frame has been allocated. - size = frame->varp - frame->sp; - if(thechar != '6' && thechar != '8') - minsize = sizeof(uintptr); - else - minsize = 0; - if(size > minsize) { - stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); - if(stackmap == nil || stackmap->n <= 0) { - runtime·printf("runtime: frame %s untyped locals %p+%p\n", runtime·funcname(f), (byte*)(frame->varp-size), size); - runtime·throw("missing stackmap"); - } - - // Locals bitmap information, scan just the pointers in locals. - if(pcdata < 0 || pcdata >= stackmap->n) { - // don't know where we are - runtime·printf("runtime: pcdata is %d and %d locals stack map entries for %s (targetpc=%p)\n", - pcdata, stackmap->n, runtime·funcname(f), targetpc); - runtime·throw("scanframe: bad symbol table"); - } - bv = runtime·stackmapdata(stackmap, pcdata); - size = (bv.n * PtrSize) / BitsPerPointer; - scanblock((byte*)(frame->varp - size), bv.n/BitsPerPointer*PtrSize, bv.bytedata); - } - - // Scan arguments. - if(frame->arglen > 0) { - if(frame->argmap != nil) - bv = *frame->argmap; - else { - stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); - if(stackmap == nil || stackmap->n <= 0) { - runtime·printf("runtime: frame %s untyped args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen); - runtime·throw("missing stackmap"); - } - if(pcdata < 0 || pcdata >= stackmap->n) { - // don't know where we are - runtime·printf("runtime: pcdata is %d and %d args stack map entries for %s (targetpc=%p)\n", - pcdata, stackmap->n, runtime·funcname(f), targetpc); - runtime·throw("scanframe: bad symbol table"); - } - bv = runtime·stackmapdata(stackmap, pcdata); - } - scanblock((byte*)frame->argp, bv.n/BitsPerPointer*PtrSize, bv.bytedata); - } - return true; -} - -static void -scanstack(G *gp) -{ - M *mp; - bool (*fn)(Stkframe*, void*); - - if(runtime·readgstatus(gp)&Gscan == 0) { - runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp)); - runtime·throw("mark - bad status"); - } - - switch(runtime·readgstatus(gp)&~Gscan) { - default: - runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp)); - runtime·throw("mark - bad status"); - case Gdead: - return; - case Grunning: - runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp)); - runtime·throw("mark - world not stopped"); - case Grunnable: - case Gsyscall: - case Gwaiting: - break; - } - - if(gp == g) - runtime·throw("can't scan our own stack"); - if((mp = gp->m) != nil && mp->helpgc) - runtime·throw("can't scan gchelper stack"); - - fn = scanframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &fn, nil, 0); - runtime·tracebackdefers(gp, &fn, nil); -} - -// The gp has been moved to a gc safepoint. If there is gcphase specific -// work it is done here. -void -runtime·gcphasework(G *gp) -{ - switch(runtime·gcphase) { - default: - runtime·throw("gcphasework in bad gcphase"); - case GCoff: - case GCquiesce: - case GCstw: - case GCsweep: - // No work for now. - break; - case GCmark: - // Disabled until concurrent GC is implemented - // but indicate the scan has been done. - // scanstack(gp); - break; - } - gp->gcworkdone = true; -} - -#pragma dataflag NOPTR -static byte finalizer1[] = { - // Each Finalizer is 5 words, ptr ptr uintptr ptr ptr. - // Each byte describes 4 words. - // Need 4 Finalizers described by 5 bytes before pattern repeats: - // ptr ptr uintptr ptr ptr - // ptr ptr uintptr ptr ptr - // ptr ptr uintptr ptr ptr - // ptr ptr uintptr ptr ptr - // aka - // ptr ptr uintptr ptr - // ptr ptr ptr uintptr - // ptr ptr ptr ptr - // uintptr ptr ptr ptr - // ptr uintptr ptr ptr - // Assumptions about Finalizer layout checked below. - BitsPointer | BitsPointer<<2 | BitsScalar<<4 | BitsPointer<<6, - BitsPointer | BitsPointer<<2 | BitsPointer<<4 | BitsScalar<<6, - BitsPointer | BitsPointer<<2 | BitsPointer<<4 | BitsPointer<<6, - BitsScalar | BitsPointer<<2 | BitsPointer<<4 | BitsPointer<<6, - BitsPointer | BitsScalar<<2 | BitsPointer<<4 | BitsPointer<<6, -}; - -void -runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot) -{ - FinBlock *block; - Finalizer *f; - int32 i; - - runtime·lock(&runtime·finlock); - if(runtime·finq == nil || runtime·finq->cnt == runtime·finq->cap) { - if(runtime·finc == nil) { - runtime·finc = runtime·persistentalloc(FinBlockSize, 0, &mstats.gc_sys); - runtime·finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1; - runtime·finc->alllink = runtime·allfin; - runtime·allfin = runtime·finc; - if(finptrmask[0] == 0) { - // Build pointer mask for Finalizer array in block. - // Check assumptions made in finalizer1 array above. - if(sizeof(Finalizer) != 5*PtrSize || - offsetof(Finalizer, fn) != 0 || - offsetof(Finalizer, arg) != PtrSize || - offsetof(Finalizer, nret) != 2*PtrSize || - offsetof(Finalizer, fint) != 3*PtrSize || - offsetof(Finalizer, ot) != 4*PtrSize || - BitsPerPointer != 2) { - runtime·throw("finalizer out of sync"); - } - for(i=0; i<nelem(finptrmask); i++) - finptrmask[i] = finalizer1[i%nelem(finalizer1)]; - } - } - block = runtime·finc; - runtime·finc = block->next; - block->next = runtime·finq; - runtime·finq = block; - } - f = &runtime·finq->fin[runtime·finq->cnt]; - runtime·finq->cnt++; - f->fn = fn; - f->nret = nret; - f->fint = fint; - f->ot = ot; - f->arg = p; - runtime·fingwake = true; - runtime·unlock(&runtime·finlock); -} - -void -runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*)) -{ - FinBlock *fb; - Finalizer *f; - uintptr i; - - for(fb = runtime·allfin; fb; fb = fb->alllink) { - for(i = 0; i < fb->cnt; i++) { - f = &fb->fin[i]; - callback(f->fn, f->arg, f->nret, f->fint, f->ot); - } - } -} - -void -runtime·MSpan_EnsureSwept(MSpan *s) -{ - uint32 sg; - - // Caller must disable preemption. - // Otherwise when this function returns the span can become unswept again - // (if GC is triggered on another goroutine). - if(g->m->locks == 0 && g->m->mallocing == 0 && g != g->m->g0) - runtime·throw("MSpan_EnsureSwept: m is not locked"); - - sg = runtime·mheap.sweepgen; - if(runtime·atomicload(&s->sweepgen) == sg) - return; - if(runtime·cas(&s->sweepgen, sg-2, sg-1)) { - runtime·MSpan_Sweep(s, false); - return; - } - // unfortunate condition, and we don't have efficient means to wait - while(runtime·atomicload(&s->sweepgen) != sg) - runtime·osyield(); -} - -// Sweep frees or collects finalizers for blocks not marked in the mark phase. -// It clears the mark bits in preparation for the next GC round. -// Returns true if the span was returned to heap. -// If preserve=true, don't return it to heap nor relink in MCentral lists; -// caller takes care of it. -bool -runtime·MSpan_Sweep(MSpan *s, bool preserve) -{ - int32 cl, n, npages, nfree; - uintptr size, off, step; - uint32 sweepgen; - byte *p, *bitp, shift, xbits, bits; - MCache *c; - byte *arena_start; - MLink head, *end, *link; - Special *special, **specialp, *y; - bool res, sweepgenset; - - // It's critical that we enter this function with preemption disabled, - // GC must not start while we are in the middle of this function. - if(g->m->locks == 0 && g->m->mallocing == 0 && g != g->m->g0) - runtime·throw("MSpan_Sweep: m is not locked"); - sweepgen = runtime·mheap.sweepgen; - if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) { - runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n", - s->state, s->sweepgen, sweepgen); - runtime·throw("MSpan_Sweep: bad span state"); - } - arena_start = runtime·mheap.arena_start; - cl = s->sizeclass; - size = s->elemsize; - if(cl == 0) { - n = 1; - } else { - // Chunk full of small blocks. - npages = runtime·class_to_allocnpages[cl]; - n = (npages << PageShift) / size; - } - res = false; - nfree = 0; - end = &head; - c = g->m->mcache; - sweepgenset = false; - - // Mark any free objects in this span so we don't collect them. - for(link = s->freelist; link != nil; link = link->next) { - off = (uintptr*)link - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - *bitp |= bitMarked<<shift; - } - - // Unlink & free special records for any objects we're about to free. - specialp = &s->specials; - special = *specialp; - while(special != nil) { - // A finalizer can be set for an inner byte of an object, find object beginning. - p = (byte*)(s->start << PageShift) + special->offset/size*size; - off = (uintptr*)p - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - bits = (*bitp>>shift) & bitMask; - if((bits&bitMarked) == 0) { - // Find the exact byte for which the special was setup - // (as opposed to object beginning). - p = (byte*)(s->start << PageShift) + special->offset; - // about to free object: splice out special record - y = special; - special = special->next; - *specialp = special; - if(!runtime·freespecial(y, p, size, false)) { - // stop freeing of object if it has a finalizer - *bitp |= bitMarked << shift; - } - } else { - // object is still live: keep special record - specialp = &special->next; - special = *specialp; - } - } - - // Sweep through n objects of given size starting at p. - // This thread owns the span now, so it can manipulate - // the block bitmap without atomic operations. - p = (byte*)(s->start << PageShift); - // Find bits for the beginning of the span. - off = (uintptr*)p - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = 0; - step = size/(PtrSize*wordsPerBitmapByte); - // Rewind to the previous quadruple as we move to the next - // in the beginning of the loop. - bitp += step; - if(step == 0) { - // 8-byte objects. - bitp++; - shift = gcBits; - } - for(; n > 0; n--, p += size) { - bitp -= step; - if(step == 0) { - if(shift != 0) - bitp--; - shift = gcBits - shift; - } - - xbits = *bitp; - bits = (xbits>>shift) & bitMask; - - // Allocated and marked object, reset bits to allocated. - if((bits&bitMarked) != 0) { - *bitp &= ~(bitMarked<<shift); - continue; - } - // At this point we know that we are looking at garbage object - // that needs to be collected. - if(runtime·debug.allocfreetrace) - runtime·tracefree(p, size); - // Reset to allocated+noscan. - *bitp = (xbits & ~((bitMarked|(BitsMask<<2))<<shift)) | ((uintptr)BitsDead<<(shift+2)); - if(cl == 0) { - // Free large span. - if(preserve) - runtime·throw("can't preserve large span"); - runtime·unmarkspan(p, s->npages<<PageShift); - s->needzero = 1; - // important to set sweepgen before returning it to heap - runtime·atomicstore(&s->sweepgen, sweepgen); - sweepgenset = true; - // NOTE(rsc,dvyukov): The original implementation of efence - // in CL 22060046 used SysFree instead of SysFault, so that - // the operating system would eventually give the memory - // back to us again, so that an efence program could run - // longer without running out of memory. Unfortunately, - // calling SysFree here without any kind of adjustment of the - // heap data structures means that when the memory does - // come back to us, we have the wrong metadata for it, either in - // the MSpan structures or in the garbage collection bitmap. - // Using SysFault here means that the program will run out of - // memory fairly quickly in efence mode, but at least it won't - // have mysterious crashes due to confused memory reuse. - // It should be possible to switch back to SysFree if we also - // implement and then call some kind of MHeap_DeleteSpan. - if(runtime·debug.efence) { - s->limit = nil; // prevent mlookup from finding this span - runtime·SysFault(p, size); - } else - runtime·MHeap_Free(&runtime·mheap, s, 1); - c->local_nlargefree++; - c->local_largefree += size; - runtime·xadd64(&mstats.next_gc, -(uint64)(size * (runtime·gcpercent + 100)/100)); - res = true; - } else { - // Free small object. - if(size > 2*sizeof(uintptr)) - ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll; // mark as "needs to be zeroed" - else if(size > sizeof(uintptr)) - ((uintptr*)p)[1] = 0; - - end->next = (MLink*)p; - end = (MLink*)p; - nfree++; - } - } - - // We need to set s->sweepgen = h->sweepgen only when all blocks are swept, - // because of the potential for a concurrent free/SetFinalizer. - // But we need to set it before we make the span available for allocation - // (return it to heap or mcentral), because allocation code assumes that a - // span is already swept if available for allocation. - - if(!sweepgenset && nfree == 0) { - // The span must be in our exclusive ownership until we update sweepgen, - // check for potential races. - if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) { - runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n", - s->state, s->sweepgen, sweepgen); - runtime·throw("MSpan_Sweep: bad span state after sweep"); - } - runtime·atomicstore(&s->sweepgen, sweepgen); - } - if(nfree > 0) { - c->local_nsmallfree[cl] += nfree; - c->local_cachealloc -= nfree * size; - runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (runtime·gcpercent + 100)/100)); - res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl].mcentral, s, nfree, head.next, end, preserve); - // MCentral_FreeSpan updates sweepgen - } - return res; -} - -// State of background runtime·sweep. -// Protected by runtime·gclock. -typedef struct SweepData SweepData; -struct SweepData -{ - G* g; - bool parked; - - uint32 spanidx; // background sweeper position - - uint32 nbgsweep; - uint32 npausesweep; -}; -SweepData runtime·sweep; - -// sweeps one span -// returns number of pages returned to heap, or -1 if there is nothing to sweep -uintptr -runtime·sweepone(void) -{ - MSpan *s; - uint32 idx, sg; - uintptr npages; - - // increment locks to ensure that the goroutine is not preempted - // in the middle of sweep thus leaving the span in an inconsistent state for next GC - g->m->locks++; - sg = runtime·mheap.sweepgen; - for(;;) { - idx = runtime·xadd(&runtime·sweep.spanidx, 1) - 1; - if(idx >= runtime·work.nspan) { - runtime·mheap.sweepdone = true; - g->m->locks--; - return -1; - } - s = runtime·work.spans[idx]; - if(s->state != MSpanInUse) { - s->sweepgen = sg; - continue; - } - if(s->sweepgen != sg-2 || !runtime·cas(&s->sweepgen, sg-2, sg-1)) - continue; - npages = s->npages; - if(!runtime·MSpan_Sweep(s, false)) - npages = 0; - g->m->locks--; - return npages; - } -} - -static void -sweepone_m(void) -{ - g->m->scalararg[0] = runtime·sweepone(); -} - -#pragma textflag NOSPLIT -uintptr -runtime·gosweepone(void) -{ - void (*fn)(void); - - fn = sweepone_m; - runtime·onM(&fn); - return g->m->scalararg[0]; -} - -#pragma textflag NOSPLIT -bool -runtime·gosweepdone(void) -{ - return runtime·mheap.sweepdone; -} - -void -runtime·gchelper(void) -{ - uint32 nproc; - - g->m->traceback = 2; - gchelperstart(); - - // parallel mark for over gc roots - runtime·parfordo(runtime·work.markfor); - - // help other threads scan secondary blocks - scanblock(nil, 0, nil); - - nproc = runtime·work.nproc; // runtime·work.nproc can change right after we increment runtime·work.ndone - if(runtime·xadd(&runtime·work.ndone, +1) == nproc-1) - runtime·notewakeup(&runtime·work.alldone); - g->m->traceback = 0; -} - -static void -cachestats(void) -{ - MCache *c; - P *p, **pp; - - for(pp=runtime·allp; p=*pp; pp++) { - c = p->mcache; - if(c==nil) - continue; - runtime·purgecachedstats(c); - } -} - -static void -flushallmcaches(void) -{ - P *p, **pp; - MCache *c; - - // Flush MCache's to MCentral. - for(pp=runtime·allp; p=*pp; pp++) { - c = p->mcache; - if(c==nil) - continue; - runtime·MCache_ReleaseAll(c); - runtime·stackcache_clear(c); - } -} - -static void -flushallmcaches_m(G *gp) -{ - flushallmcaches(); - runtime·gogo(&gp->sched); -} - -void -runtime·updatememstats(GCStats *stats) -{ - M *mp; - MSpan *s; - int32 i; - uint64 smallfree; - uint64 *src, *dst; - void (*fn)(G*); - - if(stats) - runtime·memclr((byte*)stats, sizeof(*stats)); - for(mp=runtime·allm; mp; mp=mp->alllink) { - if(stats) { - src = (uint64*)&mp->gcstats; - dst = (uint64*)stats; - for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) - dst[i] += src[i]; - runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats)); - } - } - mstats.mcache_inuse = runtime·mheap.cachealloc.inuse; - mstats.mspan_inuse = runtime·mheap.spanalloc.inuse; - mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys + - mstats.mcache_sys + mstats.buckhash_sys + mstats.gc_sys + mstats.other_sys; - - // Calculate memory allocator stats. - // During program execution we only count number of frees and amount of freed memory. - // Current number of alive object in the heap and amount of alive heap memory - // are calculated by scanning all spans. - // Total number of mallocs is calculated as number of frees plus number of alive objects. - // Similarly, total amount of allocated memory is calculated as amount of freed memory - // plus amount of alive heap memory. - mstats.alloc = 0; - mstats.total_alloc = 0; - mstats.nmalloc = 0; - mstats.nfree = 0; - for(i = 0; i < nelem(mstats.by_size); i++) { - mstats.by_size[i].nmalloc = 0; - mstats.by_size[i].nfree = 0; - } - - // Flush MCache's to MCentral. - if(g == g->m->g0) - flushallmcaches(); - else { - fn = flushallmcaches_m; - runtime·mcall(&fn); - } - - // Aggregate local stats. - cachestats(); - - // Scan all spans and count number of alive objects. - runtime·lock(&runtime·mheap.lock); - for(i = 0; i < runtime·mheap.nspan; i++) { - s = runtime·mheap.allspans[i]; - if(s->state != MSpanInUse) - continue; - if(s->sizeclass == 0) { - mstats.nmalloc++; - mstats.alloc += s->elemsize; - } else { - mstats.nmalloc += s->ref; - mstats.by_size[s->sizeclass].nmalloc += s->ref; - mstats.alloc += s->ref*s->elemsize; - } - } - runtime·unlock(&runtime·mheap.lock); - - // Aggregate by size class. - smallfree = 0; - mstats.nfree = runtime·mheap.nlargefree; - for(i = 0; i < nelem(mstats.by_size); i++) { - mstats.nfree += runtime·mheap.nsmallfree[i]; - mstats.by_size[i].nfree = runtime·mheap.nsmallfree[i]; - mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i]; - smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i]; - } - mstats.nfree += mstats.tinyallocs; - mstats.nmalloc += mstats.nfree; - - // Calculate derived stats. - mstats.total_alloc = mstats.alloc + runtime·mheap.largefree + smallfree; - mstats.heap_alloc = mstats.alloc; - mstats.heap_objects = mstats.nmalloc - mstats.nfree; -} - -// Structure of arguments passed to function gc(). -// This allows the arguments to be passed via runtime·mcall. -struct gc_args -{ - int64 start_time; // start time of GC in ns (just before stoptheworld) - bool eagersweep; -}; - -static void gc(struct gc_args *args); - -int32 -runtime·readgogc(void) -{ - byte *p; - - p = runtime·getenv("GOGC"); - if(p == nil || p[0] == '\0') - return 100; - if(runtime·strcmp(p, (byte*)"off") == 0) - return -1; - return runtime·atoi(p); -} - -void -runtime·gcinit(void) -{ - if(sizeof(Workbuf) != WorkbufSize) - runtime·throw("runtime: size of Workbuf is suboptimal"); - - runtime·work.markfor = runtime·parforalloc(MaxGcproc); - runtime·gcpercent = runtime·readgogc(); - runtime·gcdatamask = unrollglobgcprog(runtime·gcdata, runtime·edata - runtime·data); - runtime·gcbssmask = unrollglobgcprog(runtime·gcbss, runtime·ebss - runtime·bss); -} - -void -runtime·gc_m(void) -{ - struct gc_args a; - G *gp; - - gp = g->m->curg; - runtime·casgstatus(gp, Grunning, Gwaiting); - gp->waitreason = runtime·gostringnocopy((byte*)"garbage collection"); - - a.start_time = (uint64)(g->m->scalararg[0]) | ((uint64)(g->m->scalararg[1]) << 32); - a.eagersweep = g->m->scalararg[2]; - gc(&a); - - if(nbadblock > 0) { - // Work out path from root to bad block. - for(;;) { - gc(&a); - if(nbadblock >= nelem(badblock)) - runtime·throw("cannot find path to bad pointer"); - } - } - - runtime·casgstatus(gp, Gwaiting, Grunning); -} - -static void -gc(struct gc_args *args) -{ - int64 t0, t1, t2, t3, t4; - uint64 heap0, heap1, obj; - GCStats stats; - - if(DebugPtrs) - runtime·printf("GC start\n"); - - if(runtime·debug.allocfreetrace) - runtime·tracegc(); - - g->m->traceback = 2; - t0 = args->start_time; - runtime·work.tstart = args->start_time; - - t1 = 0; - if(runtime·debug.gctrace) - t1 = runtime·nanotime(); - - // Sweep what is not sweeped by bgsweep. - while(runtime·sweepone() != -1) - runtime·sweep.npausesweep++; - - // Cache runtime.mheap.allspans in work.spans to avoid conflicts with - // resizing/freeing allspans. - // New spans can be created while GC progresses, but they are not garbage for - // this round: - // - new stack spans can be created even while the world is stopped. - // - new malloc spans can be created during the concurrent sweep - - // Even if this is stop-the-world, a concurrent exitsyscall can allocate a stack from heap. - runtime·lock(&runtime·mheap.lock); - // Free the old cached sweep array if necessary. - if(runtime·work.spans != nil && runtime·work.spans != runtime·mheap.allspans) - runtime·SysFree(runtime·work.spans, runtime·work.nspan*sizeof(runtime·work.spans[0]), &mstats.other_sys); - // Cache the current array for marking. - runtime·mheap.gcspans = runtime·mheap.allspans; - runtime·work.spans = runtime·mheap.allspans; - runtime·work.nspan = runtime·mheap.nspan; - runtime·unlock(&runtime·mheap.lock); - - runtime·work.nwait = 0; - runtime·work.ndone = 0; - runtime·work.nproc = runtime·gcprocs(); - runtime·parforsetup(runtime·work.markfor, runtime·work.nproc, RootCount + runtime·allglen, nil, false, markroot); - if(runtime·work.nproc > 1) { - runtime·noteclear(&runtime·work.alldone); - runtime·helpgc(runtime·work.nproc); - } - - t2 = 0; - if(runtime·debug.gctrace) - t2 = runtime·nanotime(); - - gchelperstart(); - runtime·parfordo(runtime·work.markfor); - scanblock(nil, 0, nil); - - t3 = 0; - if(runtime·debug.gctrace) - t3 = runtime·nanotime(); - - if(runtime·work.nproc > 1) - runtime·notesleep(&runtime·work.alldone); - - runtime·shrinkfinish(); - - cachestats(); - // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap - // estimate what was live heap size after previous GC (for tracing only) - heap0 = mstats.next_gc*100/(runtime·gcpercent+100); - // conservatively set next_gc to high value assuming that everything is live - // concurrent/lazy sweep will reduce this number while discovering new garbage - mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*runtime·gcpercent/100; - - t4 = runtime·nanotime(); - runtime·atomicstore64(&mstats.last_gc, runtime·unixnanotime()); // must be Unix time to make sense to user - mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; - mstats.pause_end[mstats.numgc%nelem(mstats.pause_end)] = t4; - mstats.pause_total_ns += t4 - t0; - mstats.numgc++; - if(mstats.debuggc) - runtime·printf("pause %D\n", t4-t0); - - if(runtime·debug.gctrace) { - heap1 = mstats.heap_alloc; - runtime·updatememstats(&stats); - if(heap1 != mstats.heap_alloc) { - runtime·printf("runtime: mstats skew: heap=%D/%D\n", heap1, mstats.heap_alloc); - runtime·throw("mstats skew"); - } - obj = mstats.nmalloc - mstats.nfree; - - stats.nprocyield += runtime·work.markfor->nprocyield; - stats.nosyield += runtime·work.markfor->nosyield; - stats.nsleep += runtime·work.markfor->nsleep; - - runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects," - " %d goroutines," - " %d/%d/%d sweeps," - " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n", - mstats.numgc, runtime·work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000, - heap0>>20, heap1>>20, obj, - mstats.nmalloc, mstats.nfree, - runtime·gcount(), - runtime·work.nspan, runtime·sweep.nbgsweep, runtime·sweep.npausesweep, - stats.nhandoff, stats.nhandoffcnt, - runtime·work.markfor->nsteal, runtime·work.markfor->nstealcnt, - stats.nprocyield, stats.nosyield, stats.nsleep); - runtime·sweep.nbgsweep = runtime·sweep.npausesweep = 0; - } - - // See the comment in the beginning of this function as to why we need the following. - // Even if this is still stop-the-world, a concurrent exitsyscall can allocate a stack from heap. - runtime·lock(&runtime·mheap.lock); - // Free the old cached mark array if necessary. - if(runtime·work.spans != nil && runtime·work.spans != runtime·mheap.allspans) - runtime·SysFree(runtime·work.spans, runtime·work.nspan*sizeof(runtime·work.spans[0]), &mstats.other_sys); - // Cache the current array for sweeping. - runtime·mheap.gcspans = runtime·mheap.allspans; - runtime·mheap.sweepgen += 2; - runtime·mheap.sweepdone = false; - runtime·work.spans = runtime·mheap.allspans; - runtime·work.nspan = runtime·mheap.nspan; - runtime·sweep.spanidx = 0; - runtime·unlock(&runtime·mheap.lock); - - if(ConcurrentSweep && !args->eagersweep) { - runtime·lock(&runtime·gclock); - if(runtime·sweep.g == nil) - runtime·sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, gc); - else if(runtime·sweep.parked) { - runtime·sweep.parked = false; - runtime·ready(runtime·sweep.g); - } - runtime·unlock(&runtime·gclock); - } else { - // Sweep all spans eagerly. - while(runtime·sweepone() != -1) - runtime·sweep.npausesweep++; - // Do an additional mProf_GC, because all 'free' events are now real as well. - runtime·mProf_GC(); - } - - runtime·mProf_GC(); - g->m->traceback = 0; - - if(DebugPtrs) - runtime·printf("GC end\n"); -} - -extern uintptr runtime·sizeof_C_MStats; - -static void readmemstats_m(void); - -void -runtime·readmemstats_m(void) -{ - MStats *stats; - - stats = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - - runtime·updatememstats(nil); - // Size of the trailing by_size array differs between Go and C, - // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. - runtime·memmove(stats, &mstats, runtime·sizeof_C_MStats); - - // Stack numbers are part of the heap numbers, separate those out for user consumption - stats->stacks_sys = stats->stacks_inuse; - stats->heap_inuse -= stats->stacks_inuse; - stats->heap_sys -= stats->stacks_inuse; -} - -static void readgcstats_m(void); - -#pragma textflag NOSPLIT -void -runtime∕debug·readGCStats(Slice *pauses) -{ - void (*fn)(void); - - g->m->ptrarg[0] = pauses; - fn = readgcstats_m; - runtime·onM(&fn); -} - -static void -readgcstats_m(void) -{ - Slice *pauses; - uint64 *p; - uint32 i, j, n; - - pauses = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - - // Calling code in runtime/debug should make the slice large enough. - if(pauses->cap < nelem(mstats.pause_ns)+3) - runtime·throw("runtime: short slice passed to readGCStats"); - - // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns. - p = (uint64*)pauses->array; - runtime·lock(&runtime·mheap.lock); - - n = mstats.numgc; - if(n > nelem(mstats.pause_ns)) - n = nelem(mstats.pause_ns); - - // The pause buffer is circular. The most recent pause is at - // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward - // from there to go back farther in time. We deliver the times - // most recent first (in p[0]). - for(i=0; i<n; i++) { - j = (mstats.numgc-1-i)%nelem(mstats.pause_ns); - p[i] = mstats.pause_ns[j]; - p[n+i] = mstats.pause_end[j]; - } - - p[n+n] = mstats.last_gc; - p[n+n+1] = mstats.numgc; - p[n+n+2] = mstats.pause_total_ns; - runtime·unlock(&runtime·mheap.lock); - pauses->len = n+n+3; -} - -void -runtime·setgcpercent_m(void) -{ - int32 in; - int32 out; - - in = (int32)(intptr)g->m->scalararg[0]; - - runtime·lock(&runtime·mheap.lock); - out = runtime·gcpercent; - if(in < 0) - in = -1; - runtime·gcpercent = in; - runtime·unlock(&runtime·mheap.lock); - - g->m->scalararg[0] = (uintptr)(intptr)out; -} - -static void -gchelperstart(void) -{ - if(g->m->helpgc < 0 || g->m->helpgc >= MaxGcproc) - runtime·throw("gchelperstart: bad m->helpgc"); - if(g != g->m->g0) - runtime·throw("gchelper not running on g0 stack"); -} - -G* -runtime·wakefing(void) -{ - G *res; - - res = nil; - runtime·lock(&runtime·finlock); - if(runtime·fingwait && runtime·fingwake) { - runtime·fingwait = false; - runtime·fingwake = false; - res = runtime·fing; - } - runtime·unlock(&runtime·finlock); - return res; -} - -// Recursively unrolls GC program in prog. -// mask is where to store the result. -// ppos is a pointer to position in mask, in bits. -// sparse says to generate 4-bits per word mask for heap (2-bits for data/bss otherwise). -static byte* -unrollgcprog1(byte *mask, byte *prog, uintptr *ppos, bool inplace, bool sparse) -{ - uintptr pos, siz, i, off; - byte *arena_start, *prog1, v, *bitp, shift; - - arena_start = runtime·mheap.arena_start; - pos = *ppos; - for(;;) { - switch(prog[0]) { - case insData: - prog++; - siz = prog[0]; - prog++; - for(i = 0; i < siz; i++) { - v = prog[i/PointersPerByte]; - v >>= (i%PointersPerByte)*BitsPerPointer; - v &= BitsMask; - if(inplace) { - // Store directly into GC bitmap. - off = (uintptr*)(mask+pos) - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - if(shift==0) - *bitp = 0; - *bitp |= v<<(shift+2); - pos += PtrSize; - } else if(sparse) { - // 4-bits per word - v <<= (pos%8)+2; - mask[pos/8] |= v; - pos += gcBits; - } else { - // 2-bits per word - v <<= pos%8; - mask[pos/8] |= v; - pos += BitsPerPointer; - } - } - prog += ROUND(siz*BitsPerPointer, 8)/8; - break; - case insArray: - prog++; - siz = 0; - for(i = 0; i < PtrSize; i++) - siz = (siz<<8) + prog[PtrSize-i-1]; - prog += PtrSize; - prog1 = nil; - for(i = 0; i < siz; i++) - prog1 = unrollgcprog1(mask, prog, &pos, inplace, sparse); - if(prog1[0] != insArrayEnd) - runtime·throw("unrollgcprog: array does not end with insArrayEnd"); - prog = prog1+1; - break; - case insArrayEnd: - case insEnd: - *ppos = pos; - return prog; - default: - runtime·throw("unrollgcprog: unknown instruction"); - } - } -} - -// Unrolls GC program prog for data/bss, returns dense GC mask. -static BitVector -unrollglobgcprog(byte *prog, uintptr size) -{ - byte *mask; - uintptr pos, masksize; - - masksize = ROUND(ROUND(size, PtrSize)/PtrSize*BitsPerPointer, 8)/8; - mask = runtime·persistentalloc(masksize+1, 0, &mstats.gc_sys); - mask[masksize] = 0xa1; - pos = 0; - prog = unrollgcprog1(mask, prog, &pos, false, false); - if(pos != size/PtrSize*BitsPerPointer) { - runtime·printf("unrollglobgcprog: bad program size, got %D, expect %D\n", - (uint64)pos, (uint64)size/PtrSize*BitsPerPointer); - runtime·throw("unrollglobgcprog: bad program size"); - } - if(prog[0] != insEnd) - runtime·throw("unrollglobgcprog: program does not end with insEnd"); - if(mask[masksize] != 0xa1) - runtime·throw("unrollglobgcprog: overflow"); - return (BitVector){masksize*8, mask}; -} - -void -runtime·unrollgcproginplace_m(void) -{ - uintptr size, size0, pos, off; - byte *arena_start, *prog, *bitp, shift; - Type *typ; - void *v; - - v = g->m->ptrarg[0]; - typ = g->m->ptrarg[1]; - size = g->m->scalararg[0]; - size0 = g->m->scalararg[1]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - - pos = 0; - prog = (byte*)typ->gc[1]; - while(pos != size0) - unrollgcprog1(v, prog, &pos, true, true); - // Mark first word as bitAllocated. - arena_start = runtime·mheap.arena_start; - off = (uintptr*)v - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - *bitp |= bitBoundary<<shift; - // Mark word after last as BitsDead. - if(size0 < size) { - off = (uintptr*)((byte*)v + size0) - (uintptr*)arena_start; - bitp = arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - *bitp &= ~(bitPtrMask<<shift) | ((uintptr)BitsDead<<(shift+2)); - } -} - -// Unrolls GC program in typ->gc[1] into typ->gc[0] -void -runtime·unrollgcprog_m(void) -{ - static Mutex lock; - Type *typ; - byte *mask, *prog; - uintptr pos; - uint32 x; - - typ = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - - runtime·lock(&lock); - mask = (byte*)typ->gc[0]; - if(mask[0] == 0) { - pos = 8; // skip the unroll flag - prog = (byte*)typ->gc[1]; - prog = unrollgcprog1(mask, prog, &pos, false, true); - if(prog[0] != insEnd) - runtime·throw("unrollgcprog: program does not end with insEnd"); - if(((typ->size/PtrSize)%2) != 0) { - // repeat the program twice - prog = (byte*)typ->gc[1]; - unrollgcprog1(mask, prog, &pos, false, true); - } - // atomic way to say mask[0] = 1 - x = ((uint32*)mask)[0]; - runtime·atomicstore((uint32*)mask, x|1); - } - runtime·unlock(&lock); -} - -// mark the span of memory at v as having n blocks of the given size. -// if leftover is true, there is left over space at the end of the span. -void -runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) -{ - uintptr i, off, step; - byte *b; - - if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start) - runtime·throw("markspan: bad pointer"); - - // Find bits of the beginning of the span. - off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset - b = runtime·mheap.arena_start - off/wordsPerBitmapByte - 1; - if((off%wordsPerBitmapByte) != 0) - runtime·throw("markspan: unaligned length"); - - // Okay to use non-atomic ops here, because we control - // the entire span, and each bitmap byte has bits for only - // one span, so no other goroutines are changing these bitmap words. - - if(size == PtrSize) { - // Possible only on 64-bits (minimal size class is 8 bytes). - // Poor man's memset(0x11). - if(0x11 != ((bitBoundary+BitsDead)<<gcBits) + (bitBoundary+BitsDead)) - runtime·throw("markspan: bad bits"); - if((n%(wordsPerBitmapByte*PtrSize)) != 0) - runtime·throw("markspan: unaligned length"); - b = b - n/wordsPerBitmapByte + 1; // find first byte - if(((uintptr)b%PtrSize) != 0) - runtime·throw("markspan: unaligned pointer"); - for(i = 0; i != n; i += wordsPerBitmapByte*PtrSize, b += PtrSize) - *(uintptr*)b = (uintptr)0x1111111111111111ULL; // bitBoundary+BitsDead - return; - } - - if(leftover) - n++; // mark a boundary just past end of last block too - step = size/(PtrSize*wordsPerBitmapByte); - for(i = 0; i != n; i++, b -= step) - *b = bitBoundary|(BitsDead<<2); -} - -// unmark the span of memory at v of length n bytes. -void -runtime·unmarkspan(void *v, uintptr n) -{ - uintptr off; - byte *b; - - if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start) - runtime·throw("markspan: bad pointer"); - - off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset - if((off % (PtrSize*wordsPerBitmapByte)) != 0) - runtime·throw("markspan: unaligned pointer"); - b = runtime·mheap.arena_start - off/wordsPerBitmapByte - 1; - n /= PtrSize; - if(n%(PtrSize*wordsPerBitmapByte) != 0) - runtime·throw("unmarkspan: unaligned length"); - // Okay to use non-atomic ops here, because we control - // the entire span, and each bitmap word has bits for only - // one span, so no other goroutines are changing these - // bitmap words. - n /= wordsPerBitmapByte; - runtime·memclr(b - n + 1, n); -} - -void -runtime·MHeap_MapBits(MHeap *h) -{ - // Caller has added extra mappings to the arena. - // Add extra mappings of bitmap words as needed. - // We allocate extra bitmap pieces in chunks of bitmapChunk. - enum { - bitmapChunk = 8192 - }; - uintptr n; - - n = (h->arena_used - h->arena_start) / (PtrSize*wordsPerBitmapByte); - n = ROUND(n, bitmapChunk); - n = ROUND(n, PhysPageSize); - if(h->bitmap_mapped >= n) - return; - - runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats.gc_sys); - h->bitmap_mapped = n; -} - -static bool -getgcmaskcb(Stkframe *frame, void *ctxt) -{ - Stkframe *frame0; - - frame0 = ctxt; - if(frame->sp <= frame0->sp && frame0->sp < frame->varp) { - *frame0 = *frame; - return false; - } - return true; -} - -// Returns GC type info for object p for testing. -void -runtime·getgcmask(byte *p, Type *t, byte **mask, uintptr *len) -{ - Stkframe frame; - uintptr i, n, off; - byte *base, bits, shift, *b; - bool (*cb)(Stkframe*, void*); - - *mask = nil; - *len = 0; - - // data - if(p >= runtime·data && p < runtime·edata) { - n = ((PtrType*)t)->elem->size; - *len = n/PtrSize; - *mask = runtime·mallocgc(*len, nil, FlagNoScan); - for(i = 0; i < n; i += PtrSize) { - off = (p+i-runtime·data)/PtrSize; - bits = (runtime·gcdatamask.bytedata[off/PointersPerByte] >> ((off%PointersPerByte)*BitsPerPointer))&BitsMask; - (*mask)[i/PtrSize] = bits; - } - return; - } - // bss - if(p >= runtime·bss && p < runtime·ebss) { - n = ((PtrType*)t)->elem->size; - *len = n/PtrSize; - *mask = runtime·mallocgc(*len, nil, FlagNoScan); - for(i = 0; i < n; i += PtrSize) { - off = (p+i-runtime·bss)/PtrSize; - bits = (runtime·gcbssmask.bytedata[off/PointersPerByte] >> ((off%PointersPerByte)*BitsPerPointer))&BitsMask; - (*mask)[i/PtrSize] = bits; - } - return; - } - // heap - if(runtime·mlookup(p, &base, &n, nil)) { - *len = n/PtrSize; - *mask = runtime·mallocgc(*len, nil, FlagNoScan); - for(i = 0; i < n; i += PtrSize) { - off = (uintptr*)(base+i) - (uintptr*)runtime·mheap.arena_start; - b = runtime·mheap.arena_start - off/wordsPerBitmapByte - 1; - shift = (off % wordsPerBitmapByte) * gcBits; - bits = (*b >> (shift+2))&BitsMask; - (*mask)[i/PtrSize] = bits; - } - return; - } - // stack - frame.fn = nil; - frame.sp = (uintptr)p; - cb = getgcmaskcb; - runtime·gentraceback(g->m->curg->sched.pc, g->m->curg->sched.sp, 0, g->m->curg, 0, nil, 1000, &cb, &frame, 0); - if(frame.fn != nil) { - Func *f; - StackMap *stackmap; - BitVector bv; - uintptr size; - uintptr targetpc; - int32 pcdata; - - f = frame.fn; - targetpc = frame.continpc; - if(targetpc == 0) - return; - if(targetpc != f->entry) - targetpc--; - pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); - if(pcdata == -1) - return; - stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); - if(stackmap == nil || stackmap->n <= 0) - return; - bv = runtime·stackmapdata(stackmap, pcdata); - size = bv.n/BitsPerPointer*PtrSize; - n = ((PtrType*)t)->elem->size; - *len = n/PtrSize; - *mask = runtime·mallocgc(*len, nil, FlagNoScan); - for(i = 0; i < n; i += PtrSize) { - off = (p+i-(byte*)frame.varp+size)/PtrSize; - bits = (bv.bytedata[off*BitsPerPointer/8] >> ((off*BitsPerPointer)%8))&BitsMask; - (*mask)[i/PtrSize] = bits; - } - } -} - -void runtime·gc_unixnanotime(int64 *now); - -int64 -runtime·unixnanotime(void) -{ - int64 now; - - runtime·gc_unixnanotime(&now); - return now; -} diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go index cbf5e9cfd..38406f33a 100644 --- a/src/runtime/mgc0.go +++ b/src/runtime/mgc0.go @@ -28,7 +28,7 @@ func gc_unixnanotime(now *int64) { func freeOSMemory() { gogc(2) // force GC and do eager sweep - onM(scavenge_m) + systemstack(scavenge_m) } var poolcleanup func() @@ -76,10 +76,8 @@ func clearpools() { } } -func gosweepone() uintptr -func gosweepdone() bool - func bgsweep() { + sweep.g = getg() getg().issystem = true for { for gosweepone() != ^uintptr(0) { diff --git a/src/runtime/mgc0.h b/src/runtime/mgc0.h index 64f818914..62726b4f0 100644 --- a/src/runtime/mgc0.h +++ b/src/runtime/mgc0.h @@ -2,77 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Garbage collector (GC) +// Used by cmd/gc. enum { - // Four bits per word (see #defines below). gcBits = 4, - wordsPerBitmapByte = 8/gcBits, - - // GC type info programs. - // The programs allow to store type info required for GC in a compact form. - // Most importantly arrays take O(1) space instead of O(n). - // The program grammar is: - // - // Program = {Block} "insEnd" - // Block = Data | Array - // Data = "insData" DataSize DataBlock - // DataSize = int // size of the DataBlock in bit pairs, 1 byte - // DataBlock = binary // dense GC mask (2 bits per word) of size ]DataSize/4[ bytes - // Array = "insArray" ArrayLen Block "insArrayEnd" - // ArrayLen = int // length of the array, 8 bytes (4 bytes for 32-bit arch) - // - // Each instruction (insData, insArray, etc) is 1 byte. - // For example, for type struct { x []byte; y [20]struct{ z int; w *byte }; } - // the program looks as: - // - // insData 3 (BitsMultiWord BitsSlice BitsScalar) - // insArray 20 insData 2 (BitsScalar BitsPointer) insArrayEnd insEnd - // - // Total size of the program is 17 bytes (13 bytes on 32-bits). - // The corresponding GC mask would take 43 bytes (it would be repeated - // because the type has odd number of words). + BitsPerPointer = 2, + BitsDead = 0, + BitsScalar = 1, + BitsPointer = 2, + BitsMask = 3, + PointersPerByte = 8/BitsPerPointer, + MaxGCMask = 64, insData = 1, insArray, insArrayEnd, insEnd, - - // Pointer map - BitsPerPointer = 2, - BitsMask = (1<<BitsPerPointer)-1, - PointersPerByte = 8/BitsPerPointer, - - // If you change these, also change scanblock. - // scanblock does "if(bits == BitsScalar || bits == BitsDead)" as "if(bits <= BitsScalar)". - BitsDead = 0, - BitsScalar = 1, - BitsPointer = 2, - BitsMultiWord = 3, - // BitsMultiWord will be set for the first word of a multi-word item. - // When it is set, one of the following will be set for the second word. - // NOT USED ANYMORE: BitsString = 0, - // NOT USED ANYMORE: BitsSlice = 1, - BitsIface = 2, - BitsEface = 3, - - // 64 bytes cover objects of size 1024/512 on 64/32 bits, respectively. - MaxGCMask = 64, -}; - -// Bits in per-word bitmap. -// #defines because we shift the values beyond 32 bits. -// -// Each word in the bitmap describes wordsPerBitmapWord words -// of heap memory. There are 4 bitmap bits dedicated to each heap word, -// so on a 64-bit system there is one bitmap word per 16 heap words. -// -// The bitmap starts at mheap.arena_start and extends *backward* from -// there. On a 64-bit system the off'th word in the arena is tracked by -// the off/16+1'th word before mheap.arena_start. (On a 32-bit system, -// the only difference is that the divisor is 8.) -enum { - bitBoundary = 1, // boundary of an object - bitMarked = 2, // marked object - bitMask = bitBoundary | bitMarked, - bitPtrMask = BitsMask<<2, }; diff --git a/src/runtime/mgc1.go b/src/runtime/mgc1.go new file mode 100644 index 000000000..d1aab4554 --- /dev/null +++ b/src/runtime/mgc1.go @@ -0,0 +1,77 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector (GC) + +package runtime + +const ( + // Four bits per word (see #defines below). + gcBits = 4 + wordsPerBitmapByte = 8 / gcBits +) + +const ( + // GC type info programs. + // The programs allow to store type info required for GC in a compact form. + // Most importantly arrays take O(1) space instead of O(n). + // The program grammar is: + // + // Program = {Block} "insEnd" + // Block = Data | Array + // Data = "insData" DataSize DataBlock + // DataSize = int // size of the DataBlock in bit pairs, 1 byte + // DataBlock = binary // dense GC mask (2 bits per word) of size ]DataSize/4[ bytes + // Array = "insArray" ArrayLen Block "insArrayEnd" + // ArrayLen = int // length of the array, 8 bytes (4 bytes for 32-bit arch) + // + // Each instruction (insData, insArray, etc) is 1 byte. + // For example, for type struct { x []byte; y [20]struct{ z int; w *byte }; } + // the program looks as: + // + // insData 3 (BitsPointer BitsScalar BitsScalar) + // insArray 20 insData 2 (BitsScalar BitsPointer) insArrayEnd insEnd + // + // Total size of the program is 17 bytes (13 bytes on 32-bits). + // The corresponding GC mask would take 43 bytes (it would be repeated + // because the type has odd number of words). + insData = 1 + iota + insArray + insArrayEnd + insEnd +) + +const ( + // Pointer map + _BitsPerPointer = 2 + _BitsMask = (1 << _BitsPerPointer) - 1 + _PointersPerByte = 8 / _BitsPerPointer + + // If you change these, also change scanblock. + // scanblock does "if(bits == BitsScalar || bits == BitsDead)" as "if(bits <= BitsScalar)". + _BitsDead = 0 + _BitsScalar = 1 + _BitsPointer = 2 + + // 64 bytes cover objects of size 1024/512 on 64/32 bits, respectively. + _MaxGCMask = 64 +) + +// Bits in per-word bitmap. +// #defines because we shift the values beyond 32 bits. +// +// Each word in the bitmap describes wordsPerBitmapWord words +// of heap memory. There are 4 bitmap bits dedicated to each heap word, +// so on a 64-bit system there is one bitmap word per 16 heap words. +// +// The bitmap starts at mheap.arena_start and extends *backward* from +// there. On a 64-bit system the off'th word in the arena is tracked by +// the off/16+1'th word before mheap.arena_start. (On a 32-bit system, +// the only difference is that the divisor is 8.) +const ( + bitBoundary = 1 // boundary of an object + bitMarked = 2 // marked object + bitMask = bitBoundary | bitMarked + bitPtrMask = _BitsMask << 2 +) diff --git a/src/runtime/mheap.c b/src/runtime/mheap.c deleted file mode 100644 index bb203d5ce..000000000 --- a/src/runtime/mheap.c +++ /dev/null @@ -1,889 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Page heap. -// -// See malloc.h for overview. -// -// When a MSpan is in the heap free list, state == MSpanFree -// and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. -// -// When a MSpan is allocated, state == MSpanInUse or MSpanStack -// and heapmap(i) == span for all s->start <= i < s->start+s->npages. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -static MSpan *MHeap_AllocSpanLocked(MHeap*, uintptr); -static void MHeap_FreeSpanLocked(MHeap*, MSpan*, bool, bool); -static bool MHeap_Grow(MHeap*, uintptr); -static MSpan *MHeap_AllocLarge(MHeap*, uintptr); -static MSpan *BestFit(MSpan*, uintptr, MSpan*); - -static void -RecordSpan(void *vh, byte *p) -{ - MHeap *h; - MSpan *s; - MSpan **all; - uint32 cap; - - h = vh; - s = (MSpan*)p; - if(h->nspan >= h->nspancap) { - cap = 64*1024/sizeof(all[0]); - if(cap < h->nspancap*3/2) - cap = h->nspancap*3/2; - all = (MSpan**)runtime·sysAlloc(cap*sizeof(all[0]), &mstats.other_sys); - if(all == nil) - runtime·throw("runtime: cannot allocate memory"); - if(h->allspans) { - runtime·memmove(all, h->allspans, h->nspancap*sizeof(all[0])); - // Don't free the old array if it's referenced by sweep. - // See the comment in mgc0.c. - if(h->allspans != runtime·mheap.gcspans) - runtime·SysFree(h->allspans, h->nspancap*sizeof(all[0]), &mstats.other_sys); - } - h->allspans = all; - h->nspancap = cap; - } - h->allspans[h->nspan++] = s; -} - -// Initialize the heap; fetch memory using alloc. -void -runtime·MHeap_Init(MHeap *h) -{ - uint32 i; - - runtime·FixAlloc_Init(&h->spanalloc, sizeof(MSpan), RecordSpan, h, &mstats.mspan_sys); - runtime·FixAlloc_Init(&h->cachealloc, sizeof(MCache), nil, nil, &mstats.mcache_sys); - runtime·FixAlloc_Init(&h->specialfinalizeralloc, sizeof(SpecialFinalizer), nil, nil, &mstats.other_sys); - runtime·FixAlloc_Init(&h->specialprofilealloc, sizeof(SpecialProfile), nil, nil, &mstats.other_sys); - // h->mapcache needs no init - for(i=0; i<nelem(h->free); i++) { - runtime·MSpanList_Init(&h->free[i]); - runtime·MSpanList_Init(&h->busy[i]); - } - runtime·MSpanList_Init(&h->freelarge); - runtime·MSpanList_Init(&h->busylarge); - for(i=0; i<nelem(h->central); i++) - runtime·MCentral_Init(&h->central[i].mcentral, i); -} - -void -runtime·MHeap_MapSpans(MHeap *h) -{ - uintptr n; - - // Map spans array, PageSize at a time. - n = (uintptr)h->arena_used; - n -= (uintptr)h->arena_start; - n = n / PageSize * sizeof(h->spans[0]); - n = ROUND(n, PhysPageSize); - if(h->spans_mapped >= n) - return; - runtime·SysMap((byte*)h->spans + h->spans_mapped, n - h->spans_mapped, h->arena_reserved, &mstats.other_sys); - h->spans_mapped = n; -} - -// Sweeps spans in list until reclaims at least npages into heap. -// Returns the actual number of pages reclaimed. -static uintptr -MHeap_ReclaimList(MHeap *h, MSpan *list, uintptr npages) -{ - MSpan *s; - uintptr n; - uint32 sg; - - n = 0; - sg = runtime·mheap.sweepgen; -retry: - for(s = list->next; s != list; s = s->next) { - if(s->sweepgen == sg-2 && runtime·cas(&s->sweepgen, sg-2, sg-1)) { - runtime·MSpanList_Remove(s); - // swept spans are at the end of the list - runtime·MSpanList_InsertBack(list, s); - runtime·unlock(&h->lock); - n += runtime·MSpan_Sweep(s, false); - runtime·lock(&h->lock); - if(n >= npages) - return n; - // the span could have been moved elsewhere - goto retry; - } - if(s->sweepgen == sg-1) { - // the span is being sweept by background sweeper, skip - continue; - } - // already swept empty span, - // all subsequent ones must also be either swept or in process of sweeping - break; - } - return n; -} - -// Sweeps and reclaims at least npage pages into heap. -// Called before allocating npage pages. -static void -MHeap_Reclaim(MHeap *h, uintptr npage) -{ - uintptr reclaimed, n; - - // First try to sweep busy spans with large objects of size >= npage, - // this has good chances of reclaiming the necessary space. - for(n=npage; n < nelem(h->busy); n++) { - if(MHeap_ReclaimList(h, &h->busy[n], npage)) - return; // Bingo! - } - - // Then -- even larger objects. - if(MHeap_ReclaimList(h, &h->busylarge, npage)) - return; // Bingo! - - // Now try smaller objects. - // One such object is not enough, so we need to reclaim several of them. - reclaimed = 0; - for(n=0; n < npage && n < nelem(h->busy); n++) { - reclaimed += MHeap_ReclaimList(h, &h->busy[n], npage-reclaimed); - if(reclaimed >= npage) - return; - } - - // Now sweep everything that is not yet swept. - runtime·unlock(&h->lock); - for(;;) { - n = runtime·sweepone(); - if(n == -1) // all spans are swept - break; - reclaimed += n; - if(reclaimed >= npage) - break; - } - runtime·lock(&h->lock); -} - -// Allocate a new span of npage pages from the heap for GC'd memory -// and record its size class in the HeapMap and HeapMapCache. -static MSpan* -mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large) -{ - MSpan *s; - - if(g != g->m->g0) - runtime·throw("mheap_alloc not on M stack"); - runtime·lock(&h->lock); - - // To prevent excessive heap growth, before allocating n pages - // we need to sweep and reclaim at least n pages. - if(!h->sweepdone) - MHeap_Reclaim(h, npage); - - // transfer stats from cache to global - mstats.heap_alloc += g->m->mcache->local_cachealloc; - g->m->mcache->local_cachealloc = 0; - mstats.tinyallocs += g->m->mcache->local_tinyallocs; - g->m->mcache->local_tinyallocs = 0; - - s = MHeap_AllocSpanLocked(h, npage); - if(s != nil) { - // Record span info, because gc needs to be - // able to map interior pointer to containing span. - runtime·atomicstore(&s->sweepgen, h->sweepgen); - s->state = MSpanInUse; - s->freelist = nil; - s->ref = 0; - s->sizeclass = sizeclass; - s->elemsize = (sizeclass==0 ? s->npages<<PageShift : runtime·class_to_size[sizeclass]); - - // update stats, sweep lists - if(large) { - mstats.heap_objects++; - mstats.heap_alloc += npage<<PageShift; - // Swept spans are at the end of lists. - if(s->npages < nelem(h->free)) - runtime·MSpanList_InsertBack(&h->busy[s->npages], s); - else - runtime·MSpanList_InsertBack(&h->busylarge, s); - } - } - runtime·unlock(&h->lock); - return s; -} - -static void -mheap_alloc_m(G *gp) -{ - MHeap *h; - MSpan *s; - - h = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - s = mheap_alloc(h, g->m->scalararg[0], g->m->scalararg[1], g->m->scalararg[2]); - g->m->ptrarg[0] = s; - - runtime·gogo(&gp->sched); -} - -MSpan* -runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero) -{ - MSpan *s; - void (*fn)(G*); - - // Don't do any operations that lock the heap on the G stack. - // It might trigger stack growth, and the stack growth code needs - // to be able to allocate heap. - if(g == g->m->g0) { - s = mheap_alloc(h, npage, sizeclass, large); - } else { - g->m->ptrarg[0] = h; - g->m->scalararg[0] = npage; - g->m->scalararg[1] = sizeclass; - g->m->scalararg[2] = large; - fn = mheap_alloc_m; - runtime·mcall(&fn); - s = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - } - if(s != nil) { - if(needzero && s->needzero) - runtime·memclr((byte*)(s->start<<PageShift), s->npages<<PageShift); - s->needzero = 0; - } - return s; -} - -MSpan* -runtime·MHeap_AllocStack(MHeap *h, uintptr npage) -{ - MSpan *s; - - if(g != g->m->g0) - runtime·throw("mheap_allocstack not on M stack"); - runtime·lock(&h->lock); - s = MHeap_AllocSpanLocked(h, npage); - if(s != nil) { - s->state = MSpanStack; - s->freelist = nil; - s->ref = 0; - mstats.stacks_inuse += s->npages<<PageShift; - } - runtime·unlock(&h->lock); - return s; -} - -// Allocates a span of the given size. h must be locked. -// The returned span has been removed from the -// free list, but its state is still MSpanFree. -static MSpan* -MHeap_AllocSpanLocked(MHeap *h, uintptr npage) -{ - uintptr n; - MSpan *s, *t; - pageID p; - - // Try in fixed-size lists up to max. - for(n=npage; n < nelem(h->free); n++) { - if(!runtime·MSpanList_IsEmpty(&h->free[n])) { - s = h->free[n].next; - goto HaveSpan; - } - } - - // Best fit in list of large spans. - if((s = MHeap_AllocLarge(h, npage)) == nil) { - if(!MHeap_Grow(h, npage)) - return nil; - if((s = MHeap_AllocLarge(h, npage)) == nil) - return nil; - } - -HaveSpan: - // Mark span in use. - if(s->state != MSpanFree) - runtime·throw("MHeap_AllocLocked - MSpan not free"); - if(s->npages < npage) - runtime·throw("MHeap_AllocLocked - bad npages"); - runtime·MSpanList_Remove(s); - if(s->next != nil || s->prev != nil) - runtime·throw("still in list"); - if(s->npreleased > 0) { - runtime·SysUsed((void*)(s->start<<PageShift), s->npages<<PageShift); - mstats.heap_released -= s->npreleased<<PageShift; - s->npreleased = 0; - } - - if(s->npages > npage) { - // Trim extra and put it back in the heap. - t = runtime·FixAlloc_Alloc(&h->spanalloc); - runtime·MSpan_Init(t, s->start + npage, s->npages - npage); - s->npages = npage; - p = t->start; - p -= ((uintptr)h->arena_start>>PageShift); - if(p > 0) - h->spans[p-1] = s; - h->spans[p] = t; - h->spans[p+t->npages-1] = t; - t->needzero = s->needzero; - s->state = MSpanStack; // prevent coalescing with s - t->state = MSpanStack; - MHeap_FreeSpanLocked(h, t, false, false); - t->unusedsince = s->unusedsince; // preserve age (TODO: wrong: t is possibly merged and/or deallocated at this point) - s->state = MSpanFree; - } - s->unusedsince = 0; - - p = s->start; - p -= ((uintptr)h->arena_start>>PageShift); - for(n=0; n<npage; n++) - h->spans[p+n] = s; - - mstats.heap_inuse += npage<<PageShift; - mstats.heap_idle -= npage<<PageShift; - - //runtime·printf("spanalloc %p\n", s->start << PageShift); - if(s->next != nil || s->prev != nil) - runtime·throw("still in list"); - return s; -} - -// Allocate a span of exactly npage pages from the list of large spans. -static MSpan* -MHeap_AllocLarge(MHeap *h, uintptr npage) -{ - return BestFit(&h->freelarge, npage, nil); -} - -// Search list for smallest span with >= npage pages. -// If there are multiple smallest spans, take the one -// with the earliest starting address. -static MSpan* -BestFit(MSpan *list, uintptr npage, MSpan *best) -{ - MSpan *s; - - for(s=list->next; s != list; s=s->next) { - if(s->npages < npage) - continue; - if(best == nil - || s->npages < best->npages - || (s->npages == best->npages && s->start < best->start)) - best = s; - } - return best; -} - -// Try to add at least npage pages of memory to the heap, -// returning whether it worked. -static bool -MHeap_Grow(MHeap *h, uintptr npage) -{ - uintptr ask; - void *v; - MSpan *s; - pageID p; - - // Ask for a big chunk, to reduce the number of mappings - // the operating system needs to track; also amortizes - // the overhead of an operating system mapping. - // Allocate a multiple of 64kB. - npage = ROUND(npage, (64<<10)/PageSize); - ask = npage<<PageShift; - if(ask < HeapAllocChunk) - ask = HeapAllocChunk; - - v = runtime·MHeap_SysAlloc(h, ask); - if(v == nil) { - if(ask > (npage<<PageShift)) { - ask = npage<<PageShift; - v = runtime·MHeap_SysAlloc(h, ask); - } - if(v == nil) { - runtime·printf("runtime: out of memory: cannot allocate %D-byte block (%D in use)\n", (uint64)ask, mstats.heap_sys); - return false; - } - } - - // Create a fake "in use" span and free it, so that the - // right coalescing happens. - s = runtime·FixAlloc_Alloc(&h->spanalloc); - runtime·MSpan_Init(s, (uintptr)v>>PageShift, ask>>PageShift); - p = s->start; - p -= ((uintptr)h->arena_start>>PageShift); - h->spans[p] = s; - h->spans[p + s->npages - 1] = s; - runtime·atomicstore(&s->sweepgen, h->sweepgen); - s->state = MSpanInUse; - MHeap_FreeSpanLocked(h, s, false, true); - return true; -} - -// Look up the span at the given address. -// Address is guaranteed to be in map -// and is guaranteed to be start or end of span. -MSpan* -runtime·MHeap_Lookup(MHeap *h, void *v) -{ - uintptr p; - - p = (uintptr)v; - p -= (uintptr)h->arena_start; - return h->spans[p >> PageShift]; -} - -// Look up the span at the given address. -// Address is *not* guaranteed to be in map -// and may be anywhere in the span. -// Map entries for the middle of a span are only -// valid for allocated spans. Free spans may have -// other garbage in their middles, so we have to -// check for that. -MSpan* -runtime·MHeap_LookupMaybe(MHeap *h, void *v) -{ - MSpan *s; - pageID p, q; - - if((byte*)v < h->arena_start || (byte*)v >= h->arena_used) - return nil; - p = (uintptr)v>>PageShift; - q = p; - q -= (uintptr)h->arena_start >> PageShift; - s = h->spans[q]; - if(s == nil || p < s->start || v >= s->limit || s->state != MSpanInUse) - return nil; - return s; -} - -// Free the span back into the heap. -static void -mheap_free(MHeap *h, MSpan *s, int32 acct) -{ - if(g != g->m->g0) - runtime·throw("mheap_free not on M stack"); - runtime·lock(&h->lock); - mstats.heap_alloc += g->m->mcache->local_cachealloc; - g->m->mcache->local_cachealloc = 0; - mstats.tinyallocs += g->m->mcache->local_tinyallocs; - g->m->mcache->local_tinyallocs = 0; - if(acct) { - mstats.heap_alloc -= s->npages<<PageShift; - mstats.heap_objects--; - } - MHeap_FreeSpanLocked(h, s, true, true); - runtime·unlock(&h->lock); -} - -static void -mheap_free_m(G *gp) -{ - MHeap *h; - MSpan *s; - - h = g->m->ptrarg[0]; - s = g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - mheap_free(h, s, g->m->scalararg[0]); - runtime·gogo(&gp->sched); -} - -void -runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct) -{ - void (*fn)(G*); - - if(g == g->m->g0) { - mheap_free(h, s, acct); - } else { - g->m->ptrarg[0] = h; - g->m->ptrarg[1] = s; - g->m->scalararg[0] = acct; - fn = mheap_free_m; - runtime·mcall(&fn); - } -} - -void -runtime·MHeap_FreeStack(MHeap *h, MSpan *s) -{ - if(g != g->m->g0) - runtime·throw("mheap_freestack not on M stack"); - s->needzero = 1; - runtime·lock(&h->lock); - mstats.stacks_inuse -= s->npages<<PageShift; - MHeap_FreeSpanLocked(h, s, true, true); - runtime·unlock(&h->lock); -} - -static void -MHeap_FreeSpanLocked(MHeap *h, MSpan *s, bool acctinuse, bool acctidle) -{ - MSpan *t; - pageID p; - - switch(s->state) { - case MSpanStack: - if(s->ref != 0) - runtime·throw("MHeap_FreeSpanLocked - invalid stack free"); - break; - case MSpanInUse: - if(s->ref != 0 || s->sweepgen != h->sweepgen) { - runtime·printf("MHeap_FreeSpanLocked - span %p ptr %p ref %d sweepgen %d/%d\n", - s, s->start<<PageShift, s->ref, s->sweepgen, h->sweepgen); - runtime·throw("MHeap_FreeSpanLocked - invalid free"); - } - break; - default: - runtime·throw("MHeap_FreeSpanLocked - invalid span state"); - break; - } - if(acctinuse) - mstats.heap_inuse -= s->npages<<PageShift; - if(acctidle) - mstats.heap_idle += s->npages<<PageShift; - s->state = MSpanFree; - runtime·MSpanList_Remove(s); - // Stamp newly unused spans. The scavenger will use that - // info to potentially give back some pages to the OS. - s->unusedsince = runtime·nanotime(); - s->npreleased = 0; - - // Coalesce with earlier, later spans. - p = s->start; - p -= (uintptr)h->arena_start >> PageShift; - if(p > 0 && (t = h->spans[p-1]) != nil && t->state != MSpanInUse && t->state != MSpanStack) { - s->start = t->start; - s->npages += t->npages; - s->npreleased = t->npreleased; // absorb released pages - s->needzero |= t->needzero; - p -= t->npages; - h->spans[p] = s; - runtime·MSpanList_Remove(t); - t->state = MSpanDead; - runtime·FixAlloc_Free(&h->spanalloc, t); - } - if((p+s->npages)*sizeof(h->spans[0]) < h->spans_mapped && (t = h->spans[p+s->npages]) != nil && t->state != MSpanInUse && t->state != MSpanStack) { - s->npages += t->npages; - s->npreleased += t->npreleased; - s->needzero |= t->needzero; - h->spans[p + s->npages - 1] = s; - runtime·MSpanList_Remove(t); - t->state = MSpanDead; - runtime·FixAlloc_Free(&h->spanalloc, t); - } - - // Insert s into appropriate list. - if(s->npages < nelem(h->free)) - runtime·MSpanList_Insert(&h->free[s->npages], s); - else - runtime·MSpanList_Insert(&h->freelarge, s); -} - -static uintptr -scavengelist(MSpan *list, uint64 now, uint64 limit) -{ - uintptr released, sumreleased; - MSpan *s; - - if(runtime·MSpanList_IsEmpty(list)) - return 0; - - sumreleased = 0; - for(s=list->next; s != list; s=s->next) { - if((now - s->unusedsince) > limit && s->npreleased != s->npages) { - released = (s->npages - s->npreleased) << PageShift; - mstats.heap_released += released; - sumreleased += released; - s->npreleased = s->npages; - runtime·SysUnused((void*)(s->start << PageShift), s->npages << PageShift); - } - } - return sumreleased; -} - -void -runtime·MHeap_Scavenge(int32 k, uint64 now, uint64 limit) -{ - uint32 i; - uintptr sumreleased; - MHeap *h; - - h = &runtime·mheap; - runtime·lock(&h->lock); - sumreleased = 0; - for(i=0; i < nelem(h->free); i++) - sumreleased += scavengelist(&h->free[i], now, limit); - sumreleased += scavengelist(&h->freelarge, now, limit); - runtime·unlock(&h->lock); - - if(runtime·debug.gctrace > 0) { - if(sumreleased > 0) - runtime·printf("scvg%d: %D MB released\n", k, (uint64)sumreleased>>20); - // TODO(dvyukov): these stats are incorrect as we don't subtract stack usage from heap. - // But we can't call ReadMemStats on g0 holding locks. - runtime·printf("scvg%d: inuse: %D, idle: %D, sys: %D, released: %D, consumed: %D (MB)\n", - k, mstats.heap_inuse>>20, mstats.heap_idle>>20, mstats.heap_sys>>20, - mstats.heap_released>>20, (mstats.heap_sys - mstats.heap_released)>>20); - } -} - -void -runtime·scavenge_m(void) -{ - runtime·MHeap_Scavenge(-1, ~(uintptr)0, 0); -} - -// Initialize a new span with the given start and npages. -void -runtime·MSpan_Init(MSpan *span, pageID start, uintptr npages) -{ - span->next = nil; - span->prev = nil; - span->start = start; - span->npages = npages; - span->freelist = nil; - span->ref = 0; - span->sizeclass = 0; - span->incache = false; - span->elemsize = 0; - span->state = MSpanDead; - span->unusedsince = 0; - span->npreleased = 0; - span->specialLock.key = 0; - span->specials = nil; - span->needzero = 0; -} - -// Initialize an empty doubly-linked list. -void -runtime·MSpanList_Init(MSpan *list) -{ - list->state = MSpanListHead; - list->next = list; - list->prev = list; -} - -void -runtime·MSpanList_Remove(MSpan *span) -{ - if(span->prev == nil && span->next == nil) - return; - span->prev->next = span->next; - span->next->prev = span->prev; - span->prev = nil; - span->next = nil; -} - -bool -runtime·MSpanList_IsEmpty(MSpan *list) -{ - return list->next == list; -} - -void -runtime·MSpanList_Insert(MSpan *list, MSpan *span) -{ - if(span->next != nil || span->prev != nil) { - runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span->next, span->prev); - runtime·throw("MSpanList_Insert"); - } - span->next = list->next; - span->prev = list; - span->next->prev = span; - span->prev->next = span; -} - -void -runtime·MSpanList_InsertBack(MSpan *list, MSpan *span) -{ - if(span->next != nil || span->prev != nil) { - runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span->next, span->prev); - runtime·throw("MSpanList_Insert"); - } - span->next = list; - span->prev = list->prev; - span->next->prev = span; - span->prev->next = span; -} - -// Adds the special record s to the list of special records for -// the object p. All fields of s should be filled in except for -// offset & next, which this routine will fill in. -// Returns true if the special was successfully added, false otherwise. -// (The add will fail only if a record with the same p and s->kind -// already exists.) -static bool -addspecial(void *p, Special *s) -{ - MSpan *span; - Special **t, *x; - uintptr offset; - byte kind; - - span = runtime·MHeap_LookupMaybe(&runtime·mheap, p); - if(span == nil) - runtime·throw("addspecial on invalid pointer"); - - // Ensure that the span is swept. - // GC accesses specials list w/o locks. And it's just much safer. - g->m->locks++; - runtime·MSpan_EnsureSwept(span); - - offset = (uintptr)p - (span->start << PageShift); - kind = s->kind; - - runtime·lock(&span->specialLock); - - // Find splice point, check for existing record. - t = &span->specials; - while((x = *t) != nil) { - if(offset == x->offset && kind == x->kind) { - runtime·unlock(&span->specialLock); - g->m->locks--; - return false; // already exists - } - if(offset < x->offset || (offset == x->offset && kind < x->kind)) - break; - t = &x->next; - } - // Splice in record, fill in offset. - s->offset = offset; - s->next = x; - *t = s; - runtime·unlock(&span->specialLock); - g->m->locks--; - return true; -} - -// Removes the Special record of the given kind for the object p. -// Returns the record if the record existed, nil otherwise. -// The caller must FixAlloc_Free the result. -static Special* -removespecial(void *p, byte kind) -{ - MSpan *span; - Special *s, **t; - uintptr offset; - - span = runtime·MHeap_LookupMaybe(&runtime·mheap, p); - if(span == nil) - runtime·throw("removespecial on invalid pointer"); - - // Ensure that the span is swept. - // GC accesses specials list w/o locks. And it's just much safer. - g->m->locks++; - runtime·MSpan_EnsureSwept(span); - - offset = (uintptr)p - (span->start << PageShift); - - runtime·lock(&span->specialLock); - t = &span->specials; - while((s = *t) != nil) { - // This function is used for finalizers only, so we don't check for - // "interior" specials (p must be exactly equal to s->offset). - if(offset == s->offset && kind == s->kind) { - *t = s->next; - runtime·unlock(&span->specialLock); - g->m->locks--; - return s; - } - t = &s->next; - } - runtime·unlock(&span->specialLock); - g->m->locks--; - return nil; -} - -// Adds a finalizer to the object p. Returns true if it succeeded. -bool -runtime·addfinalizer(void *p, FuncVal *f, uintptr nret, Type *fint, PtrType *ot) -{ - SpecialFinalizer *s; - - runtime·lock(&runtime·mheap.speciallock); - s = runtime·FixAlloc_Alloc(&runtime·mheap.specialfinalizeralloc); - runtime·unlock(&runtime·mheap.speciallock); - s->special.kind = KindSpecialFinalizer; - s->fn = f; - s->nret = nret; - s->fint = fint; - s->ot = ot; - if(addspecial(p, &s->special)) - return true; - - // There was an old finalizer - runtime·lock(&runtime·mheap.speciallock); - runtime·FixAlloc_Free(&runtime·mheap.specialfinalizeralloc, s); - runtime·unlock(&runtime·mheap.speciallock); - return false; -} - -// Removes the finalizer (if any) from the object p. -void -runtime·removefinalizer(void *p) -{ - SpecialFinalizer *s; - - s = (SpecialFinalizer*)removespecial(p, KindSpecialFinalizer); - if(s == nil) - return; // there wasn't a finalizer to remove - runtime·lock(&runtime·mheap.speciallock); - runtime·FixAlloc_Free(&runtime·mheap.specialfinalizeralloc, s); - runtime·unlock(&runtime·mheap.speciallock); -} - -// Set the heap profile bucket associated with addr to b. -void -runtime·setprofilebucket_m(void) -{ - void *p; - Bucket *b; - SpecialProfile *s; - - p = g->m->ptrarg[0]; - b = g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - - runtime·lock(&runtime·mheap.speciallock); - s = runtime·FixAlloc_Alloc(&runtime·mheap.specialprofilealloc); - runtime·unlock(&runtime·mheap.speciallock); - s->special.kind = KindSpecialProfile; - s->b = b; - if(!addspecial(p, &s->special)) - runtime·throw("setprofilebucket: profile already set"); -} - -// Do whatever cleanup needs to be done to deallocate s. It has -// already been unlinked from the MSpan specials list. -// Returns true if we should keep working on deallocating p. -bool -runtime·freespecial(Special *s, void *p, uintptr size, bool freed) -{ - SpecialFinalizer *sf; - SpecialProfile *sp; - - switch(s->kind) { - case KindSpecialFinalizer: - sf = (SpecialFinalizer*)s; - runtime·queuefinalizer(p, sf->fn, sf->nret, sf->fint, sf->ot); - runtime·lock(&runtime·mheap.speciallock); - runtime·FixAlloc_Free(&runtime·mheap.specialfinalizeralloc, sf); - runtime·unlock(&runtime·mheap.speciallock); - return false; // don't free p until finalizer is done - case KindSpecialProfile: - sp = (SpecialProfile*)s; - runtime·mProf_Free(sp->b, size, freed); - runtime·lock(&runtime·mheap.speciallock); - runtime·FixAlloc_Free(&runtime·mheap.specialprofilealloc, sp); - runtime·unlock(&runtime·mheap.speciallock); - return true; - default: - runtime·throw("bad special kind"); - return true; - } -} diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go new file mode 100644 index 000000000..fedcd69c5 --- /dev/null +++ b/src/runtime/mheap.go @@ -0,0 +1,785 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Page heap. +// +// See malloc.h for overview. +// +// When a MSpan is in the heap free list, state == MSpanFree +// and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. +// +// When a MSpan is allocated, state == MSpanInUse or MSpanStack +// and heapmap(i) == span for all s->start <= i < s->start+s->npages. + +package runtime + +import "unsafe" + +var h_allspans []*mspan // TODO: make this h.allspans once mheap can be defined in Go +var h_spans []*mspan // TODO: make this h.spans once mheap can be defined in Go + +func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { + h := (*mheap)(vh) + s := (*mspan)(p) + if len(h_allspans) >= cap(h_allspans) { + n := 64 * 1024 / ptrSize + if n < cap(h_allspans)*3/2 { + n = cap(h_allspans) * 3 / 2 + } + var new []*mspan + sp := (*slice)(unsafe.Pointer(&new)) + sp.array = (*byte)(sysAlloc(uintptr(n)*ptrSize, &memstats.other_sys)) + if sp.array == nil { + gothrow("runtime: cannot allocate memory") + } + sp.len = uint(len(h_allspans)) + sp.cap = uint(n) + if len(h_allspans) > 0 { + copy(new, h_allspans) + // Don't free the old array if it's referenced by sweep. + // See the comment in mgc0.c. + if h.allspans != mheap_.gcspans { + sysFree(unsafe.Pointer(h.allspans), uintptr(cap(h_allspans))*ptrSize, &memstats.other_sys) + } + } + h_allspans = new + h.allspans = (**mspan)(unsafe.Pointer(sp.array)) + } + h_allspans = append(h_allspans, s) + h.nspan = uint32(len(h_allspans)) +} + +// Initialize the heap. +func mHeap_Init(h *mheap, spans_size uintptr) { + fixAlloc_Init(&h.spanalloc, unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) + fixAlloc_Init(&h.cachealloc, unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) + fixAlloc_Init(&h.specialfinalizeralloc, unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) + fixAlloc_Init(&h.specialprofilealloc, unsafe.Sizeof(specialprofile{}), nil, nil, &memstats.other_sys) + + // h->mapcache needs no init + for i := range h.free { + mSpanList_Init(&h.free[i]) + mSpanList_Init(&h.busy[i]) + } + + mSpanList_Init(&h.freelarge) + mSpanList_Init(&h.busylarge) + for i := range h.central { + mCentral_Init(&h.central[i].mcentral, int32(i)) + } + + sp := (*slice)(unsafe.Pointer(&h_spans)) + sp.array = (*byte)(unsafe.Pointer(h.spans)) + sp.len = uint(spans_size / ptrSize) + sp.cap = uint(spans_size / ptrSize) +} + +func mHeap_MapSpans(h *mheap) { + // Map spans array, PageSize at a time. + n := uintptr(unsafe.Pointer(h.arena_used)) + n -= uintptr(unsafe.Pointer(h.arena_start)) + n = n / _PageSize * ptrSize + n = round(n, _PhysPageSize) + if h.spans_mapped >= n { + return + } + sysMap(add(unsafe.Pointer(h.spans), h.spans_mapped), n-h.spans_mapped, h.arena_reserved, &memstats.other_sys) + h.spans_mapped = n +} + +// Sweeps spans in list until reclaims at least npages into heap. +// Returns the actual number of pages reclaimed. +func mHeap_ReclaimList(h *mheap, list *mspan, npages uintptr) uintptr { + n := uintptr(0) + sg := mheap_.sweepgen +retry: + for s := list.next; s != list; s = s.next { + if s.sweepgen == sg-2 && cas(&s.sweepgen, sg-2, sg-1) { + mSpanList_Remove(s) + // swept spans are at the end of the list + mSpanList_InsertBack(list, s) + unlock(&h.lock) + if mSpan_Sweep(s, false) { + // TODO(rsc,dvyukov): This is probably wrong. + // It is undercounting the number of pages reclaimed. + // See golang.org/issue/9048. + // Note that if we want to add the true count of s's pages, + // we must record that before calling mSpan_Sweep, + // because if mSpan_Sweep returns true the span has + // been + n++ + } + lock(&h.lock) + if n >= npages { + return n + } + // the span could have been moved elsewhere + goto retry + } + if s.sweepgen == sg-1 { + // the span is being sweept by background sweeper, skip + continue + } + // already swept empty span, + // all subsequent ones must also be either swept or in process of sweeping + break + } + return n +} + +// Sweeps and reclaims at least npage pages into heap. +// Called before allocating npage pages. +func mHeap_Reclaim(h *mheap, npage uintptr) { + // First try to sweep busy spans with large objects of size >= npage, + // this has good chances of reclaiming the necessary space. + for i := int(npage); i < len(h.busy); i++ { + if mHeap_ReclaimList(h, &h.busy[i], npage) != 0 { + return // Bingo! + } + } + + // Then -- even larger objects. + if mHeap_ReclaimList(h, &h.busylarge, npage) != 0 { + return // Bingo! + } + + // Now try smaller objects. + // One such object is not enough, so we need to reclaim several of them. + reclaimed := uintptr(0) + for i := 0; i < int(npage) && i < len(h.busy); i++ { + reclaimed += mHeap_ReclaimList(h, &h.busy[i], npage-reclaimed) + if reclaimed >= npage { + return + } + } + + // Now sweep everything that is not yet swept. + unlock(&h.lock) + for { + n := sweepone() + if n == ^uintptr(0) { // all spans are swept + break + } + reclaimed += n + if reclaimed >= npage { + break + } + } + lock(&h.lock) +} + +// Allocate a new span of npage pages from the heap for GC'd memory +// and record its size class in the HeapMap and HeapMapCache. +func mHeap_Alloc_m(h *mheap, npage uintptr, sizeclass int32, large bool) *mspan { + _g_ := getg() + if _g_ != _g_.m.g0 { + gothrow("_mheap_alloc not on g0 stack") + } + lock(&h.lock) + + // To prevent excessive heap growth, before allocating n pages + // we need to sweep and reclaim at least n pages. + if h.sweepdone == 0 { + mHeap_Reclaim(h, npage) + } + + // transfer stats from cache to global + memstats.heap_alloc += uint64(_g_.m.mcache.local_cachealloc) + _g_.m.mcache.local_cachealloc = 0 + memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs) + _g_.m.mcache.local_tinyallocs = 0 + + s := mHeap_AllocSpanLocked(h, npage) + if s != nil { + // Record span info, because gc needs to be + // able to map interior pointer to containing span. + atomicstore(&s.sweepgen, h.sweepgen) + s.state = _MSpanInUse + s.freelist = nil + s.ref = 0 + s.sizeclass = uint8(sizeclass) + if sizeclass == 0 { + s.elemsize = s.npages << _PageShift + } else { + s.elemsize = uintptr(class_to_size[sizeclass]) + } + + // update stats, sweep lists + if large { + memstats.heap_objects++ + memstats.heap_alloc += uint64(npage << _PageShift) + // Swept spans are at the end of lists. + if s.npages < uintptr(len(h.free)) { + mSpanList_InsertBack(&h.busy[s.npages], s) + } else { + mSpanList_InsertBack(&h.busylarge, s) + } + } + } + unlock(&h.lock) + return s +} + +func mHeap_Alloc(h *mheap, npage uintptr, sizeclass int32, large bool, needzero bool) *mspan { + // Don't do any operations that lock the heap on the G stack. + // It might trigger stack growth, and the stack growth code needs + // to be able to allocate heap. + var s *mspan + systemstack(func() { + s = mHeap_Alloc_m(h, npage, sizeclass, large) + }) + + if s != nil { + if needzero && s.needzero != 0 { + memclr(unsafe.Pointer(s.start<<_PageShift), s.npages<<_PageShift) + } + s.needzero = 0 + } + return s +} + +func mHeap_AllocStack(h *mheap, npage uintptr) *mspan { + _g_ := getg() + if _g_ != _g_.m.g0 { + gothrow("mheap_allocstack not on g0 stack") + } + lock(&h.lock) + s := mHeap_AllocSpanLocked(h, npage) + if s != nil { + s.state = _MSpanStack + s.freelist = nil + s.ref = 0 + memstats.stacks_inuse += uint64(s.npages << _PageShift) + } + unlock(&h.lock) + return s +} + +// Allocates a span of the given size. h must be locked. +// The returned span has been removed from the +// free list, but its state is still MSpanFree. +func mHeap_AllocSpanLocked(h *mheap, npage uintptr) *mspan { + var s *mspan + + // Try in fixed-size lists up to max. + for i := int(npage); i < len(h.free); i++ { + if !mSpanList_IsEmpty(&h.free[i]) { + s = h.free[i].next + goto HaveSpan + } + } + + // Best fit in list of large spans. + s = mHeap_AllocLarge(h, npage) + if s == nil { + if !mHeap_Grow(h, npage) { + return nil + } + s = mHeap_AllocLarge(h, npage) + if s == nil { + return nil + } + } + +HaveSpan: + // Mark span in use. + if s.state != _MSpanFree { + gothrow("MHeap_AllocLocked - MSpan not free") + } + if s.npages < npage { + gothrow("MHeap_AllocLocked - bad npages") + } + mSpanList_Remove(s) + if s.next != nil || s.prev != nil { + gothrow("still in list") + } + if s.npreleased > 0 { + sysUsed((unsafe.Pointer)(s.start<<_PageShift), s.npages<<_PageShift) + memstats.heap_released -= uint64(s.npreleased << _PageShift) + s.npreleased = 0 + } + + if s.npages > npage { + // Trim extra and put it back in the heap. + t := (*mspan)(fixAlloc_Alloc(&h.spanalloc)) + mSpan_Init(t, s.start+pageID(npage), s.npages-npage) + s.npages = npage + p := uintptr(t.start) + p -= (uintptr(unsafe.Pointer(h.arena_start)) >> _PageShift) + if p > 0 { + h_spans[p-1] = s + } + h_spans[p] = t + h_spans[p+t.npages-1] = t + t.needzero = s.needzero + s.state = _MSpanStack // prevent coalescing with s + t.state = _MSpanStack + mHeap_FreeSpanLocked(h, t, false, false) + t.unusedsince = s.unusedsince // preserve age (TODO: wrong: t is possibly merged and/or deallocated at this point) + s.state = _MSpanFree + } + s.unusedsince = 0 + + p := uintptr(s.start) + p -= (uintptr(unsafe.Pointer(h.arena_start)) >> _PageShift) + for n := uintptr(0); n < npage; n++ { + h_spans[p+n] = s + } + + memstats.heap_inuse += uint64(npage << _PageShift) + memstats.heap_idle -= uint64(npage << _PageShift) + + //println("spanalloc", hex(s.start<<_PageShift)) + if s.next != nil || s.prev != nil { + gothrow("still in list") + } + return s +} + +// Allocate a span of exactly npage pages from the list of large spans. +func mHeap_AllocLarge(h *mheap, npage uintptr) *mspan { + return bestFit(&h.freelarge, npage, nil) +} + +// Search list for smallest span with >= npage pages. +// If there are multiple smallest spans, take the one +// with the earliest starting address. +func bestFit(list *mspan, npage uintptr, best *mspan) *mspan { + for s := list.next; s != list; s = s.next { + if s.npages < npage { + continue + } + if best == nil || s.npages < best.npages || (s.npages == best.npages && s.start < best.start) { + best = s + } + } + return best +} + +// Try to add at least npage pages of memory to the heap, +// returning whether it worked. +func mHeap_Grow(h *mheap, npage uintptr) bool { + // Ask for a big chunk, to reduce the number of mappings + // the operating system needs to track; also amortizes + // the overhead of an operating system mapping. + // Allocate a multiple of 64kB. + npage = round(npage, (64<<10)/_PageSize) + ask := npage << _PageShift + if ask < _HeapAllocChunk { + ask = _HeapAllocChunk + } + + v := mHeap_SysAlloc(h, ask) + if v == nil { + if ask > npage<<_PageShift { + ask = npage << _PageShift + v = mHeap_SysAlloc(h, ask) + } + if v == nil { + print("runtime: out of memory: cannot allocate ", ask, "-byte block (", memstats.heap_sys, " in use)\n") + return false + } + } + + // Create a fake "in use" span and free it, so that the + // right coalescing happens. + s := (*mspan)(fixAlloc_Alloc(&h.spanalloc)) + mSpan_Init(s, pageID(uintptr(v)>>_PageShift), ask>>_PageShift) + p := uintptr(s.start) + p -= (uintptr(unsafe.Pointer(h.arena_start)) >> _PageShift) + h_spans[p] = s + h_spans[p+s.npages-1] = s + atomicstore(&s.sweepgen, h.sweepgen) + s.state = _MSpanInUse + mHeap_FreeSpanLocked(h, s, false, true) + return true +} + +// Look up the span at the given address. +// Address is guaranteed to be in map +// and is guaranteed to be start or end of span. +func mHeap_Lookup(h *mheap, v unsafe.Pointer) *mspan { + p := uintptr(v) + p -= uintptr(unsafe.Pointer(h.arena_start)) + return h_spans[p>>_PageShift] +} + +// Look up the span at the given address. +// Address is *not* guaranteed to be in map +// and may be anywhere in the span. +// Map entries for the middle of a span are only +// valid for allocated spans. Free spans may have +// other garbage in their middles, so we have to +// check for that. +func mHeap_LookupMaybe(h *mheap, v unsafe.Pointer) *mspan { + if uintptr(v) < uintptr(unsafe.Pointer(h.arena_start)) || uintptr(v) >= uintptr(unsafe.Pointer(h.arena_used)) { + return nil + } + p := uintptr(v) >> _PageShift + q := p + q -= uintptr(unsafe.Pointer(h.arena_start)) >> _PageShift + s := h_spans[q] + if s == nil || p < uintptr(s.start) || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != _MSpanInUse { + return nil + } + return s +} + +// Free the span back into the heap. +func mHeap_Free(h *mheap, s *mspan, acct int32) { + systemstack(func() { + mp := getg().m + lock(&h.lock) + memstats.heap_alloc += uint64(mp.mcache.local_cachealloc) + mp.mcache.local_cachealloc = 0 + memstats.tinyallocs += uint64(mp.mcache.local_tinyallocs) + mp.mcache.local_tinyallocs = 0 + if acct != 0 { + memstats.heap_alloc -= uint64(s.npages << _PageShift) + memstats.heap_objects-- + } + mHeap_FreeSpanLocked(h, s, true, true) + unlock(&h.lock) + }) +} + +func mHeap_FreeStack(h *mheap, s *mspan) { + _g_ := getg() + if _g_ != _g_.m.g0 { + gothrow("mheap_freestack not on g0 stack") + } + s.needzero = 1 + lock(&h.lock) + memstats.stacks_inuse -= uint64(s.npages << _PageShift) + mHeap_FreeSpanLocked(h, s, true, true) + unlock(&h.lock) +} + +func mHeap_FreeSpanLocked(h *mheap, s *mspan, acctinuse, acctidle bool) { + switch s.state { + case _MSpanStack: + if s.ref != 0 { + gothrow("MHeap_FreeSpanLocked - invalid stack free") + } + case _MSpanInUse: + if s.ref != 0 || s.sweepgen != h.sweepgen { + print("MHeap_FreeSpanLocked - span ", s, " ptr ", hex(s.start<<_PageShift), " ref ", s.ref, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") + gothrow("MHeap_FreeSpanLocked - invalid free") + } + default: + gothrow("MHeap_FreeSpanLocked - invalid span state") + } + + if acctinuse { + memstats.heap_inuse -= uint64(s.npages << _PageShift) + } + if acctidle { + memstats.heap_idle += uint64(s.npages << _PageShift) + } + s.state = _MSpanFree + mSpanList_Remove(s) + + // Stamp newly unused spans. The scavenger will use that + // info to potentially give back some pages to the OS. + s.unusedsince = nanotime() + s.npreleased = 0 + + // Coalesce with earlier, later spans. + p := uintptr(s.start) + p -= uintptr(unsafe.Pointer(h.arena_start)) >> _PageShift + if p > 0 { + t := h_spans[p-1] + if t != nil && t.state != _MSpanInUse && t.state != _MSpanStack { + s.start = t.start + s.npages += t.npages + s.npreleased = t.npreleased // absorb released pages + s.needzero |= t.needzero + p -= t.npages + h_spans[p] = s + mSpanList_Remove(t) + t.state = _MSpanDead + fixAlloc_Free(&h.spanalloc, (unsafe.Pointer)(t)) + } + } + if (p+s.npages)*ptrSize < h.spans_mapped { + t := h_spans[p+s.npages] + if t != nil && t.state != _MSpanInUse && t.state != _MSpanStack { + s.npages += t.npages + s.npreleased += t.npreleased + s.needzero |= t.needzero + h_spans[p+s.npages-1] = s + mSpanList_Remove(t) + t.state = _MSpanDead + fixAlloc_Free(&h.spanalloc, (unsafe.Pointer)(t)) + } + } + + // Insert s into appropriate list. + if s.npages < uintptr(len(h.free)) { + mSpanList_Insert(&h.free[s.npages], s) + } else { + mSpanList_Insert(&h.freelarge, s) + } +} + +func scavengelist(list *mspan, now, limit uint64) uintptr { + if mSpanList_IsEmpty(list) { + return 0 + } + + var sumreleased uintptr + for s := list.next; s != list; s = s.next { + if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages { + released := (s.npages - s.npreleased) << _PageShift + memstats.heap_released += uint64(released) + sumreleased += released + s.npreleased = s.npages + sysUnused((unsafe.Pointer)(s.start<<_PageShift), s.npages<<_PageShift) + } + } + return sumreleased +} + +func mHeap_Scavenge(k int32, now, limit uint64) { + h := &mheap_ + lock(&h.lock) + var sumreleased uintptr + for i := 0; i < len(h.free); i++ { + sumreleased += scavengelist(&h.free[i], now, limit) + } + sumreleased += scavengelist(&h.freelarge, now, limit) + unlock(&h.lock) + + if debug.gctrace > 0 { + if sumreleased > 0 { + print("scvg", k, ": ", sumreleased>>20, " MB released\n") + } + // TODO(dvyukov): these stats are incorrect as we don't subtract stack usage from heap. + // But we can't call ReadMemStats on g0 holding locks. + print("scvg", k, ": inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") + } +} + +func scavenge_m() { + mHeap_Scavenge(-1, ^uint64(0), 0) +} + +// Initialize a new span with the given start and npages. +func mSpan_Init(span *mspan, start pageID, npages uintptr) { + span.next = nil + span.prev = nil + span.start = start + span.npages = npages + span.freelist = nil + span.ref = 0 + span.sizeclass = 0 + span.incache = false + span.elemsize = 0 + span.state = _MSpanDead + span.unusedsince = 0 + span.npreleased = 0 + span.speciallock.key = 0 + span.specials = nil + span.needzero = 0 +} + +// Initialize an empty doubly-linked list. +func mSpanList_Init(list *mspan) { + list.state = _MSpanListHead + list.next = list + list.prev = list +} + +func mSpanList_Remove(span *mspan) { + if span.prev == nil && span.next == nil { + return + } + span.prev.next = span.next + span.next.prev = span.prev + span.prev = nil + span.next = nil +} + +func mSpanList_IsEmpty(list *mspan) bool { + return list.next == list +} + +func mSpanList_Insert(list *mspan, span *mspan) { + if span.next != nil || span.prev != nil { + println("failed MSpanList_Insert", span, span.next, span.prev) + gothrow("MSpanList_Insert") + } + span.next = list.next + span.prev = list + span.next.prev = span + span.prev.next = span +} + +func mSpanList_InsertBack(list *mspan, span *mspan) { + if span.next != nil || span.prev != nil { + println("failed MSpanList_InsertBack", span, span.next, span.prev) + gothrow("MSpanList_InsertBack") + } + span.next = list + span.prev = list.prev + span.next.prev = span + span.prev.next = span +} + +// Adds the special record s to the list of special records for +// the object p. All fields of s should be filled in except for +// offset & next, which this routine will fill in. +// Returns true if the special was successfully added, false otherwise. +// (The add will fail only if a record with the same p and s->kind +// already exists.) +func addspecial(p unsafe.Pointer, s *special) bool { + span := mHeap_LookupMaybe(&mheap_, p) + if span == nil { + gothrow("addspecial on invalid pointer") + } + + // Ensure that the span is swept. + // GC accesses specials list w/o locks. And it's just much safer. + mp := acquirem() + mSpan_EnsureSwept(span) + + offset := uintptr(p) - uintptr(span.start<<_PageShift) + kind := s.kind + + lock(&span.speciallock) + + // Find splice point, check for existing record. + t := &span.specials + for { + x := *t + if x == nil { + break + } + if offset == uintptr(x.offset) && kind == x.kind { + unlock(&span.speciallock) + releasem(mp) + return false // already exists + } + if offset < uintptr(x.offset) || (offset == uintptr(x.offset) && kind < x.kind) { + break + } + t = &x.next + } + + // Splice in record, fill in offset. + s.offset = uint16(offset) + s.next = *t + *t = s + unlock(&span.speciallock) + releasem(mp) + + return true +} + +// Removes the Special record of the given kind for the object p. +// Returns the record if the record existed, nil otherwise. +// The caller must FixAlloc_Free the result. +func removespecial(p unsafe.Pointer, kind uint8) *special { + span := mHeap_LookupMaybe(&mheap_, p) + if span == nil { + gothrow("removespecial on invalid pointer") + } + + // Ensure that the span is swept. + // GC accesses specials list w/o locks. And it's just much safer. + mp := acquirem() + mSpan_EnsureSwept(span) + + offset := uintptr(p) - uintptr(span.start<<_PageShift) + + lock(&span.speciallock) + t := &span.specials + for { + s := *t + if s == nil { + break + } + // This function is used for finalizers only, so we don't check for + // "interior" specials (p must be exactly equal to s->offset). + if offset == uintptr(s.offset) && kind == s.kind { + *t = s.next + unlock(&span.speciallock) + releasem(mp) + return s + } + t = &s.next + } + unlock(&span.speciallock) + releasem(mp) + return nil +} + +// Adds a finalizer to the object p. Returns true if it succeeded. +func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *ptrtype) bool { + lock(&mheap_.speciallock) + s := (*specialfinalizer)(fixAlloc_Alloc(&mheap_.specialfinalizeralloc)) + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialFinalizer + s.fn = f + s.nret = nret + s.fint = fint + s.ot = ot + if addspecial(p, &s.special) { + return true + } + + // There was an old finalizer + lock(&mheap_.speciallock) + fixAlloc_Free(&mheap_.specialfinalizeralloc, (unsafe.Pointer)(s)) + unlock(&mheap_.speciallock) + return false +} + +// Removes the finalizer (if any) from the object p. +func removefinalizer(p unsafe.Pointer) { + s := (*specialfinalizer)(unsafe.Pointer(removespecial(p, _KindSpecialFinalizer))) + if s == nil { + return // there wasn't a finalizer to remove + } + lock(&mheap_.speciallock) + fixAlloc_Free(&mheap_.specialfinalizeralloc, (unsafe.Pointer)(s)) + unlock(&mheap_.speciallock) +} + +// Set the heap profile bucket associated with addr to b. +func setprofilebucket(p unsafe.Pointer, b *bucket) { + lock(&mheap_.speciallock) + s := (*specialprofile)(fixAlloc_Alloc(&mheap_.specialprofilealloc)) + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialProfile + s.b = b + if !addspecial(p, &s.special) { + gothrow("setprofilebucket: profile already set") + } +} + +// Do whatever cleanup needs to be done to deallocate s. It has +// already been unlinked from the MSpan specials list. +// Returns true if we should keep working on deallocating p. +func freespecial(s *special, p unsafe.Pointer, size uintptr, freed bool) bool { + switch s.kind { + case _KindSpecialFinalizer: + sf := (*specialfinalizer)(unsafe.Pointer(s)) + queuefinalizer(p, sf.fn, sf.nret, sf.fint, sf.ot) + lock(&mheap_.speciallock) + fixAlloc_Free(&mheap_.specialfinalizeralloc, (unsafe.Pointer)(sf)) + unlock(&mheap_.speciallock) + return false // don't free p until finalizer is done + case _KindSpecialProfile: + sp := (*specialprofile)(unsafe.Pointer(s)) + mProf_Free(sp.b, size, freed) + lock(&mheap_.speciallock) + fixAlloc_Free(&mheap_.specialprofilealloc, (unsafe.Pointer)(sp)) + unlock(&mheap_.speciallock) + return true + default: + gothrow("bad special kind") + panic("not reached") + } +} diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index d409c6c30..ba989b1b8 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -190,8 +190,6 @@ func stkbucket(typ bucketType, size uintptr, stk []uintptr, alloc bool) *bucket return b } -func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer - func eqslice(x, y []uintptr) bool { if len(x) != len(y) { return false @@ -246,16 +244,9 @@ func mProf_Malloc(p unsafe.Pointer, size uintptr) { // This reduces potential contention and chances of deadlocks. // Since the object must be alive during call to mProf_Malloc, // it's fine to do this non-atomically. - setprofilebucket(p, b) -} - -func setprofilebucket_m() // mheap.c - -func setprofilebucket(p unsafe.Pointer, b *bucket) { - g := getg() - g.m.ptrarg[0] = p - g.m.ptrarg[1] = unsafe.Pointer(b) - onM(setprofilebucket_m) + systemstack(func() { + setprofilebucket(p, b) + }) } // Called when freeing a profiled block. @@ -519,8 +510,6 @@ func ThreadCreateProfile(p []StackRecord) (n int, ok bool) { return } -var allgs []*g // proc.c - // GoroutineProfile returns n, the number of records in the active goroutine stack profile. // If len(p) >= n, GoroutineProfile copies the profile into p and returns n, true. // If len(p) < n, GoroutineProfile does not change p and returns n, false. @@ -534,7 +523,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { gp := getg() semacquire(&worldsema, false) gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) n = NumGoroutine() if n <= len(p) { @@ -542,7 +531,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { r := p sp := getcallersp(unsafe.Pointer(&p)) pc := getcallerpc(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { saveg(pc, sp, gp, &r[0]) }) r = r[1:] @@ -557,7 +546,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { gp.m.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) } return n, ok @@ -581,7 +570,7 @@ func Stack(buf []byte, all bool) int { semacquire(&worldsema, false) mp.gcing = 1 releasem(mp) - onM(stoptheworld) + systemstack(stoptheworld) if mp != acquirem() { gothrow("Stack: rescheduled") } @@ -591,7 +580,7 @@ func Stack(buf []byte, all bool) int { if len(buf) > 0 { sp := getcallersp(unsafe.Pointer(&buf)) pc := getcallerpc(unsafe.Pointer(&buf)) - onM(func() { + systemstack(func() { g0 := getg() g0.writebuf = buf[0:0:len(buf)] goroutineheader(gp) @@ -607,7 +596,7 @@ func Stack(buf []byte, all bool) int { if all { mp.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) } releasem(mp) return n @@ -630,7 +619,7 @@ func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) { goroutineheader(gp) pc := getcallerpc(unsafe.Pointer(&p)) sp := getcallersp(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { traceback(pc, sp, 0, gp) }) } else { @@ -650,7 +639,7 @@ func tracefree(p unsafe.Pointer, size uintptr) { goroutineheader(gp) pc := getcallerpc(unsafe.Pointer(&p)) sp := getcallersp(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { traceback(pc, sp, 0, gp) }) print("\n") diff --git a/src/runtime/msize.c b/src/runtime/msize.c deleted file mode 100644 index 7cb65dad0..000000000 --- a/src/runtime/msize.c +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Malloc small size classes. -// -// See malloc.h for overview. -// -// The size classes are chosen so that rounding an allocation -// request up to the next size class wastes at most 12.5% (1.125x). -// -// Each size class has its own page count that gets allocated -// and chopped up when new objects of the size class are needed. -// That page count is chosen so that chopping up the run of -// pages into objects of the given size wastes at most 12.5% (1.125x) -// of the memory. It is not necessary that the cutoff here be -// the same as above. -// -// The two sources of waste multiply, so the worst possible case -// for the above constraints would be that allocations of some -// size might have a 26.6% (1.266x) overhead. -// In practice, only one of the wastes comes into play for a -// given size (sizes < 512 waste mainly on the round-up, -// sizes > 512 waste mainly on the page chopping). -// -// TODO(rsc): Compute max waste for any given size. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "textflag.h" - -#pragma dataflag NOPTR -int32 runtime·class_to_size[NumSizeClasses]; -#pragma dataflag NOPTR -int32 runtime·class_to_allocnpages[NumSizeClasses]; - -// The SizeToClass lookup is implemented using two arrays, -// one mapping sizes <= 1024 to their class and one mapping -// sizes >= 1024 and <= MaxSmallSize to their class. -// All objects are 8-aligned, so the first array is indexed by -// the size divided by 8 (rounded up). Objects >= 1024 bytes -// are 128-aligned, so the second array is indexed by the -// size divided by 128 (rounded up). The arrays are filled in -// by InitSizes. - -#pragma dataflag NOPTR -int8 runtime·size_to_class8[1024/8 + 1]; -#pragma dataflag NOPTR -int8 runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1]; - -void runtime·testdefersizes(void); - -int32 -runtime·SizeToClass(int32 size) -{ - if(size > MaxSmallSize) - runtime·throw("SizeToClass - invalid size"); - if(size > 1024-8) - return runtime·size_to_class128[(size-1024+127) >> 7]; - return runtime·size_to_class8[(size+7)>>3]; -} - -void -runtime·InitSizes(void) -{ - int32 align, sizeclass, size, nextsize, n; - uint32 i; - uintptr allocsize, npages; - - // Initialize the runtime·class_to_size table (and choose class sizes in the process). - runtime·class_to_size[0] = 0; - sizeclass = 1; // 0 means no class - align = 8; - for(size = align; size <= MaxSmallSize; size += align) { - if((size&(size-1)) == 0) { // bump alignment once in a while - if(size >= 2048) - align = 256; - else if(size >= 128) - align = size / 8; - else if(size >= 16) - align = 16; // required for x86 SSE instructions, if we want to use them - } - if((align&(align-1)) != 0) - runtime·throw("InitSizes - bug"); - - // Make the allocnpages big enough that - // the leftover is less than 1/8 of the total, - // so wasted space is at most 12.5%. - allocsize = PageSize; - while(allocsize%size > allocsize/8) - allocsize += PageSize; - npages = allocsize >> PageShift; - - // If the previous sizeclass chose the same - // allocation size and fit the same number of - // objects into the page, we might as well - // use just this size instead of having two - // different sizes. - if(sizeclass > 1 && - npages == runtime·class_to_allocnpages[sizeclass-1] && - allocsize/size == allocsize/runtime·class_to_size[sizeclass-1]) { - runtime·class_to_size[sizeclass-1] = size; - continue; - } - - runtime·class_to_allocnpages[sizeclass] = npages; - runtime·class_to_size[sizeclass] = size; - sizeclass++; - } - if(sizeclass != NumSizeClasses) { - runtime·printf("sizeclass=%d NumSizeClasses=%d\n", sizeclass, NumSizeClasses); - runtime·throw("InitSizes - bad NumSizeClasses"); - } - - // Initialize the size_to_class tables. - nextsize = 0; - for (sizeclass = 1; sizeclass < NumSizeClasses; sizeclass++) { - for(; nextsize < 1024 && nextsize <= runtime·class_to_size[sizeclass]; nextsize+=8) - runtime·size_to_class8[nextsize/8] = sizeclass; - if(nextsize >= 1024) - for(; nextsize <= runtime·class_to_size[sizeclass]; nextsize += 128) - runtime·size_to_class128[(nextsize-1024)/128] = sizeclass; - } - - // Double-check SizeToClass. - if(0) { - for(n=0; n < MaxSmallSize; n++) { - sizeclass = runtime·SizeToClass(n); - if(sizeclass < 1 || sizeclass >= NumSizeClasses || runtime·class_to_size[sizeclass] < n) { - runtime·printf("size=%d sizeclass=%d runtime·class_to_size=%d\n", n, sizeclass, runtime·class_to_size[sizeclass]); - runtime·printf("incorrect SizeToClass"); - goto dump; - } - if(sizeclass > 1 && runtime·class_to_size[sizeclass-1] >= n) { - runtime·printf("size=%d sizeclass=%d runtime·class_to_size=%d\n", n, sizeclass, runtime·class_to_size[sizeclass]); - runtime·printf("SizeToClass too big"); - goto dump; - } - } - } - - runtime·testdefersizes(); - - // Copy out for statistics table. - for(i=0; i<nelem(runtime·class_to_size); i++) - mstats.by_size[i].size = runtime·class_to_size[i]; - return; - -dump: - if(1){ - runtime·printf("NumSizeClasses=%d\n", NumSizeClasses); - runtime·printf("runtime·class_to_size:"); - for(sizeclass=0; sizeclass<NumSizeClasses; sizeclass++) - runtime·printf(" %d", runtime·class_to_size[sizeclass]); - runtime·printf("\n\n"); - runtime·printf("size_to_class8:"); - for(i=0; i<nelem(runtime·size_to_class8); i++) - runtime·printf(" %d=>%d(%d)\n", i*8, runtime·size_to_class8[i], - runtime·class_to_size[runtime·size_to_class8[i]]); - runtime·printf("\n"); - runtime·printf("size_to_class128:"); - for(i=0; i<nelem(runtime·size_to_class128); i++) - runtime·printf(" %d=>%d(%d)\n", i*128, runtime·size_to_class128[i], - runtime·class_to_size[runtime·size_to_class128[i]]); - runtime·printf("\n"); - } - runtime·throw("InitSizes failed"); -} - -// Returns size of the memory block that mallocgc will allocate if you ask for the size. -uintptr -runtime·roundupsize(uintptr size) -{ - if(size < MaxSmallSize) { - if(size <= 1024-8) - return runtime·class_to_size[runtime·size_to_class8[(size+7)>>3]]; - else - return runtime·class_to_size[runtime·size_to_class128[(size-1024+127) >> 7]]; - } - if(size + PageSize < size) - return size; - return ROUND(size, PageSize); -} diff --git a/src/runtime/msize.go b/src/runtime/msize.go new file mode 100644 index 000000000..aa2b43e90 --- /dev/null +++ b/src/runtime/msize.go @@ -0,0 +1,174 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Malloc small size classes. +// +// See malloc.h for overview. +// +// The size classes are chosen so that rounding an allocation +// request up to the next size class wastes at most 12.5% (1.125x). +// +// Each size class has its own page count that gets allocated +// and chopped up when new objects of the size class are needed. +// That page count is chosen so that chopping up the run of +// pages into objects of the given size wastes at most 12.5% (1.125x) +// of the memory. It is not necessary that the cutoff here be +// the same as above. +// +// The two sources of waste multiply, so the worst possible case +// for the above constraints would be that allocations of some +// size might have a 26.6% (1.266x) overhead. +// In practice, only one of the wastes comes into play for a +// given size (sizes < 512 waste mainly on the round-up, +// sizes > 512 waste mainly on the page chopping). +// +// TODO(rsc): Compute max waste for any given size. + +package runtime + +//var class_to_size [_NumSizeClasses]int32 +//var class_to_allocnpages [_NumSizeClasses]int32 + +// The SizeToClass lookup is implemented using two arrays, +// one mapping sizes <= 1024 to their class and one mapping +// sizes >= 1024 and <= MaxSmallSize to their class. +// All objects are 8-aligned, so the first array is indexed by +// the size divided by 8 (rounded up). Objects >= 1024 bytes +// are 128-aligned, so the second array is indexed by the +// size divided by 128 (rounded up). The arrays are filled in +// by InitSizes. +//var size_to_class8 [1024/8 + 1]int8 +//var size_to_class128 [(_MaxSmallSize-1024)/128 + 1]int8 + +func sizeToClass(size int32) int32 { + if size > _MaxSmallSize { + gothrow("SizeToClass - invalid size") + } + if size > 1024-8 { + return int32(size_to_class128[(size-1024+127)>>7]) + } + return int32(size_to_class8[(size+7)>>3]) +} + +func initSizes() { + // Initialize the runtime·class_to_size table (and choose class sizes in the process). + class_to_size[0] = 0 + sizeclass := 1 // 0 means no class + align := 8 + for size := align; size <= _MaxSmallSize; size += align { + if size&(size-1) == 0 { // bump alignment once in a while + if size >= 2048 { + align = 256 + } else if size >= 128 { + align = size / 8 + } else if size >= 16 { + align = 16 // required for x86 SSE instructions, if we want to use them + } + } + if align&(align-1) != 0 { + gothrow("InitSizes - bug") + } + + // Make the allocnpages big enough that + // the leftover is less than 1/8 of the total, + // so wasted space is at most 12.5%. + allocsize := _PageSize + for allocsize%size > allocsize/8 { + allocsize += _PageSize + } + npages := allocsize >> _PageShift + + // If the previous sizeclass chose the same + // allocation size and fit the same number of + // objects into the page, we might as well + // use just this size instead of having two + // different sizes. + if sizeclass > 1 && npages == int(class_to_allocnpages[sizeclass-1]) && allocsize/size == allocsize/int(class_to_size[sizeclass-1]) { + class_to_size[sizeclass-1] = int32(size) + continue + } + + class_to_allocnpages[sizeclass] = int32(npages) + class_to_size[sizeclass] = int32(size) + sizeclass++ + } + if sizeclass != _NumSizeClasses { + print("sizeclass=", sizeclass, " NumSizeClasses=", _NumSizeClasses, "\n") + gothrow("InitSizes - bad NumSizeClasses") + } + + // Initialize the size_to_class tables. + nextsize := 0 + for sizeclass = 1; sizeclass < _NumSizeClasses; sizeclass++ { + for ; nextsize < 1024 && nextsize <= int(class_to_size[sizeclass]); nextsize += 8 { + size_to_class8[nextsize/8] = int8(sizeclass) + } + if nextsize >= 1024 { + for ; nextsize <= int(class_to_size[sizeclass]); nextsize += 128 { + size_to_class128[(nextsize-1024)/128] = int8(sizeclass) + } + } + } + + // Double-check SizeToClass. + if false { + for n := int32(0); n < _MaxSmallSize; n++ { + sizeclass := sizeToClass(n) + if sizeclass < 1 || sizeclass >= _NumSizeClasses || class_to_size[sizeclass] < n { + print("size=", n, " sizeclass=", sizeclass, " runtime·class_to_size=", class_to_size[sizeclass], "\n") + print("incorrect SizeToClass\n") + goto dump + } + if sizeclass > 1 && class_to_size[sizeclass-1] >= n { + print("size=", n, " sizeclass=", sizeclass, " runtime·class_to_size=", class_to_size[sizeclass], "\n") + print("SizeToClass too big\n") + goto dump + } + } + } + + testdefersizes() + + // Copy out for statistics table. + for i := 0; i < len(class_to_size); i++ { + memstats.by_size[i].size = uint32(class_to_size[i]) + } + return + +dump: + if true { + print("NumSizeClasses=", _NumSizeClasses, "\n") + print("runtime·class_to_size:") + for sizeclass = 0; sizeclass < _NumSizeClasses; sizeclass++ { + print(" ", class_to_size[sizeclass], "") + } + print("\n\n") + print("size_to_class8:") + for i := 0; i < len(size_to_class8); i++ { + print(" ", i*8, "=>", size_to_class8[i], "(", class_to_size[size_to_class8[i]], ")\n") + } + print("\n") + print("size_to_class128:") + for i := 0; i < len(size_to_class128); i++ { + print(" ", i*128, "=>", size_to_class128[i], "(", class_to_size[size_to_class128[i]], ")\n") + } + print("\n") + } + gothrow("InitSizes failed") +} + +// Returns size of the memory block that mallocgc will allocate if you ask for the size. +func roundupsize(size uintptr) uintptr { + if size < _MaxSmallSize { + if size <= 1024-8 { + return uintptr(class_to_size[size_to_class8[(size+7)>>3]]) + } else { + return uintptr(class_to_size[size_to_class128[(size-1024+127)>>7]]) + } + } + if size+_PageSize < size { + return size + } + return round(size, _PageSize) +} diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 3456e0208..7a99f18ad 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -49,14 +49,14 @@ type pollDesc struct { lock mutex // protectes the following fields fd uintptr closing bool - seq uintptr // protects from stale timers and ready notifications - rg uintptr // pdReady, pdWait, G waiting for read or nil - rt timer // read deadline timer (set if rt.f != nil) - rd int64 // read deadline - wg uintptr // pdReady, pdWait, G waiting for write or nil - wt timer // write deadline timer - wd int64 // write deadline - user unsafe.Pointer // user settable cookie + seq uintptr // protects from stale timers and ready notifications + rg uintptr // pdReady, pdWait, G waiting for read or nil + rt timer // read deadline timer (set if rt.f != nil) + rd int64 // read deadline + wg uintptr // pdReady, pdWait, G waiting for write or nil + wt timer // write deadline timer + wd int64 // write deadline + user uint32 // user settable cookie } type pollCache struct { @@ -72,7 +72,7 @@ type pollCache struct { var pollcache pollCache func netpollServerInit() { - onM(netpollinit) + netpollinit() } func netpollOpen(fd uintptr) (*pollDesc, int) { @@ -94,9 +94,7 @@ func netpollOpen(fd uintptr) (*pollDesc, int) { unlock(&pd.lock) var errno int32 - onM(func() { - errno = netpollopen(fd, pd) - }) + errno = netpollopen(fd, pd) return pd, int(errno) } @@ -110,9 +108,7 @@ func netpollClose(pd *pollDesc) { if pd.rg != 0 && pd.rg != pdReady { gothrow("netpollClose: blocked read on closing descriptor") } - onM(func() { - netpollclose(uintptr(pd.fd)) - }) + netpollclose(uintptr(pd.fd)) pollcache.free(pd) } @@ -143,9 +139,7 @@ func netpollWait(pd *pollDesc, mode int) int { } // As for now only Solaris uses level-triggered IO. if GOOS == "solaris" { - onM(func() { - netpollarm(pd, mode) - }) + netpollarm(pd, mode) } for !netpollblock(pd, int32(mode), false) { err = netpollcheckerr(pd, int32(mode)) @@ -263,26 +257,6 @@ func netpollUnblock(pd *pollDesc) { } } -func netpollfd(pd *pollDesc) uintptr { - return pd.fd -} - -func netpolluser(pd *pollDesc) *unsafe.Pointer { - return &pd.user -} - -func netpollclosing(pd *pollDesc) bool { - return pd.closing -} - -func netpolllock(pd *pollDesc) { - lock(&pd.lock) -} - -func netpollunlock(pd *pollDesc) { - unlock(&pd.lock) -} - // make pd ready, newly runnable goroutines (if any) are returned in rg/wg func netpollready(gpp **g, pd *pollDesc, mode int32) { var rg, wg *g @@ -343,8 +317,7 @@ func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg if waitio || netpollcheckerr(pd, mode) == 0 { - f := netpollblockcommit - gopark(**(**unsafe.Pointer)(unsafe.Pointer(&f)), unsafe.Pointer(gpp), "IO wait") + gopark(netpollblockcommit, unsafe.Pointer(gpp), "IO wait") } // be careful to not lose concurrent READY notification old := xchguintptr(gpp, 0) diff --git a/src/runtime/netpoll_solaris.c b/src/runtime/netpoll_solaris.go index d422719cf..40e8a1a65 100644 --- a/src/runtime/netpoll_solaris.c +++ b/src/runtime/netpoll_solaris.go @@ -2,13 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "runtime.h" -#include "arch_GOARCH.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" +package runtime + +import "unsafe" // Solaris runtime-integrated network poller. -// +// // Solaris uses event ports for scalable network I/O. Event // ports are level-triggered, unlike epoll and kqueue which // can be configured in both level-triggered and edge-triggered @@ -18,7 +17,7 @@ // events for that descriptor. When doing this we must keep track of // what kind of events the goroutines are currently interested in, // for example a fd may be open both for reading and writing. -// +// // A description of the high level operation of this code // follows. Networking code will get a file descriptor by some means // and will register it with the netpolling mechanism by a code path @@ -29,7 +28,7 @@ // readiness notification at some point in the future. If I/O becomes // ready when nobody is listening, when we finally care about it, // nobody will tell us anymore. -// +// // Beside calling runtime·netpollopen, the networking code paths // will call runtime·netpollarm each time goroutines are interested // in doing network I/O. Because now we know what kind of I/O we @@ -39,7 +38,7 @@ // when we now call port_associate, we will unblock the main poller // loop (in runtime·netpoll) right away if the socket is actually // ready for I/O. -// +// // The main poller loop runs in its own thread waiting for events // using port_getn. When an event happens, it will tell the scheduler // about it using runtime·netpollready. Besides doing this, it must @@ -47,7 +46,7 @@ // notification with the file descriptor. Failing to do this would // mean each notification will prevent concurrent code using the // same file descriptor in parallel. -// +// // The logic dealing with re-associations is encapsulated in // runtime·netpollupdate. This function takes care to associate the // descriptor only with the subset of events that were previously @@ -56,7 +55,7 @@ // are level triggered so it would cause a busy loop. Instead, that // association is effected only by the runtime·netpollarm code path, // when Go code actually asks for I/O. -// +// // The open and arming mechanisms are serialized using the lock // inside PollDesc. This is required because the netpoll loop runs // asynchonously in respect to other Go code and by the time we get @@ -68,179 +67,157 @@ // again we know for sure we are always talking about the same file // descriptor and can safely access the data we want (the event set). -#pragma dynimport libc·fcntl fcntl "libc.so" -#pragma dynimport libc·port_create port_create "libc.so" -#pragma dynimport libc·port_associate port_associate "libc.so" -#pragma dynimport libc·port_dissociate port_dissociate "libc.so" -#pragma dynimport libc·port_getn port_getn "libc.so" -extern uintptr libc·fcntl; -extern uintptr libc·port_create; -extern uintptr libc·port_associate; -extern uintptr libc·port_dissociate; -extern uintptr libc·port_getn; - -#define errno (*g->m->perrno) +//go:cgo_import_dynamic libc_port_create port_create "libc.so" +//go:cgo_import_dynamic libc_port_associate port_associate "libc.so" +//go:cgo_import_dynamic libc_port_dissociate port_dissociate "libc.so" +//go:cgo_import_dynamic libc_port_getn port_getn "libc.so" + +//go:linkname libc_port_create libc_port_create +//go:linkname libc_port_associate libc_port_associate +//go:linkname libc_port_dissociate libc_port_dissociate +//go:linkname libc_port_getn libc_port_getn + +var ( + libc_port_create, + libc_port_associate, + libc_port_dissociate, + libc_port_getn libcFunc +) + +func errno() int32 { + return *getg().m.perrno +} -int32 -runtime·fcntl(int32 fd, int32 cmd, uintptr arg) -{ - return runtime·sysvicall3(libc·fcntl, (uintptr)fd, (uintptr)cmd, (uintptr)arg); +func fcntl(fd, cmd int32, arg uintptr) int32 { + return int32(sysvicall3(libc_fcntl, uintptr(fd), uintptr(cmd), arg)) } -int32 -runtime·port_create(void) -{ - return runtime·sysvicall0(libc·port_create); +func port_create() int32 { + return int32(sysvicall0(libc_port_create)) } -int32 -runtime·port_associate(int32 port, int32 source, uintptr object, int32 events, uintptr user) -{ - return runtime·sysvicall5(libc·port_associate, (uintptr)port, (uintptr)source, object, (uintptr)events, user); +func port_associate(port, source int32, object uintptr, events uint32, user uintptr) int32 { + return int32(sysvicall5(libc_port_associate, uintptr(port), uintptr(source), object, uintptr(events), user)) } -int32 -runtime·port_dissociate(int32 port, int32 source, uintptr object) -{ - return runtime·sysvicall3(libc·port_dissociate, (uintptr)port, (uintptr)source, object); +func port_dissociate(port, source int32, object uintptr) int32 { + return int32(sysvicall3(libc_port_dissociate, uintptr(port), uintptr(source), object)) } -int32 -runtime·port_getn(int32 port, PortEvent *evs, uint32 max, uint32 *nget, Timespec *timeout) -{ - return runtime·sysvicall5(libc·port_getn, (uintptr)port, (uintptr)evs, (uintptr)max, (uintptr)nget, (uintptr)timeout); +func port_getn(port int32, evs *portevent, max uint32, nget *uint32, timeout *timespec) int32 { + return int32(sysvicall5(libc_port_getn, uintptr(port), uintptr(unsafe.Pointer(evs)), uintptr(max), uintptr(unsafe.Pointer(nget)), uintptr(unsafe.Pointer(timeout)))) } -static int32 portfd = -1; +var portfd int32 = -1 -void -runtime·netpollinit(void) -{ - if((portfd = runtime·port_create()) >= 0) { - runtime·fcntl(portfd, F_SETFD, FD_CLOEXEC); - return; +func netpollinit() { + portfd = port_create() + if portfd >= 0 { + fcntl(portfd, _F_SETFD, _FD_CLOEXEC) + return } - runtime·printf("netpollinit: failed to create port (%d)\n", errno); - runtime·throw("netpollinit: failed to create port"); + print("netpollinit: failed to create port (", errno(), ")\n") + gothrow("netpollinit: failed to create port") } -int32 -runtime·netpollopen(uintptr fd, PollDesc *pd) -{ - int32 r; - - runtime·netpolllock(pd); +func netpollopen(fd uintptr, pd *pollDesc) int32 { + lock(&pd.lock) // We don't register for any specific type of events yet, that's // netpollarm's job. We merely ensure we call port_associate before // asynchonous connect/accept completes, so when we actually want // to do any I/O, the call to port_associate (from netpollarm, // with the interested event set) will unblock port_getn right away // because of the I/O readiness notification. - *runtime·netpolluser(pd) = 0; - r = runtime·port_associate(portfd, PORT_SOURCE_FD, fd, 0, (uintptr)pd); - runtime·netpollunlock(pd); - return r; + pd.user = 0 + r := port_associate(portfd, _PORT_SOURCE_FD, fd, 0, uintptr(unsafe.Pointer(pd))) + unlock(&pd.lock) + return r } -int32 -runtime·netpollclose(uintptr fd) -{ - return runtime·port_dissociate(portfd, PORT_SOURCE_FD, fd); +func netpollclose(fd uintptr) int32 { + return port_dissociate(portfd, _PORT_SOURCE_FD, fd) } // Updates the association with a new set of interested events. After // this call, port_getn will return one and only one event for that // particular descriptor, so this function needs to be called again. -void -runtime·netpollupdate(PollDesc* pd, uint32 set, uint32 clear) -{ - uint32 *ep, old, events; - uintptr fd = runtime·netpollfd(pd); - ep = (uint32*)runtime·netpolluser(pd); - - if(runtime·netpollclosing(pd)) - return; +func netpollupdate(pd *pollDesc, set, clear uint32) { + if pd.closing { + return + } - old = *ep; - events = (old & ~clear) | set; - if(old == events) - return; + old := pd.user + events := (old & ^clear) | set + if old == events { + return + } - if(events && runtime·port_associate(portfd, PORT_SOURCE_FD, fd, events, (uintptr)pd) != 0) { - runtime·printf("netpollupdate: failed to associate (%d)\n", errno); - runtime·throw("netpollupdate: failed to associate"); - } - *ep = events; + if events != 0 && port_associate(portfd, _PORT_SOURCE_FD, pd.fd, events, uintptr(unsafe.Pointer(pd))) != 0 { + print("netpollupdate: failed to associate (", errno(), ")\n") + gothrow("netpollupdate: failed to associate") + } + pd.user = events } // subscribe the fd to the port such that port_getn will return one event. -void -runtime·netpollarm(PollDesc* pd, int32 mode) -{ - runtime·netpolllock(pd); - switch(mode) { +func netpollarm(pd *pollDesc, mode int) { + lock(&pd.lock) + switch mode { case 'r': - runtime·netpollupdate(pd, POLLIN, 0); - break; + netpollupdate(pd, _POLLIN, 0) case 'w': - runtime·netpollupdate(pd, POLLOUT, 0); - break; + netpollupdate(pd, _POLLOUT, 0) default: - runtime·throw("netpollarm: bad mode"); + gothrow("netpollarm: bad mode") } - runtime·netpollunlock(pd); + unlock(&pd.lock) } +// netpolllasterr holds the last error code returned by port_getn to prevent log spamming +var netpolllasterr int32 + // polls for ready network connections // returns list of goroutines that become runnable -G* -runtime·netpoll(bool block) -{ - static int32 lasterr; - PortEvent events[128], *ev; - PollDesc *pd; - int32 i, mode, clear; - uint32 n; - Timespec *wait = nil, zero; - G *gp; - - if(portfd == -1) - return (nil); +func netpoll(block bool) (gp *g) { + if portfd == -1 { + return + } - if(!block) { - zero.tv_sec = 0; - zero.tv_nsec = 0; - wait = &zero; + var wait *timespec + var zero timespec + if !block { + wait = &zero } + var events [128]portevent retry: - n = 1; - if(runtime·port_getn(portfd, events, nelem(events), &n, wait) < 0) { - if(errno != EINTR && errno != lasterr) { - lasterr = errno; - runtime·printf("runtime: port_getn on fd %d failed with %d\n", portfd, errno); + var n uint32 = 1 + if port_getn(portfd, &events[0], uint32(len(events)), &n, wait) < 0 { + if e := errno(); e != _EINTR && e != netpolllasterr { + netpolllasterr = e + print("runtime: port_getn on fd ", portfd, " failed with ", e, "\n") } - goto retry; + goto retry } - gp = nil; - for(i = 0; i < n; i++) { - ev = &events[i]; + gp = nil + for i := 0; i < int(n); i++ { + ev := &events[i] - if(ev->portev_events == 0) - continue; - pd = (PollDesc *)ev->portev_user; + if ev.portev_events == 0 { + continue + } + pd := (*pollDesc)(unsafe.Pointer(ev.portev_user)) - mode = 0; - clear = 0; - if(ev->portev_events & (POLLIN|POLLHUP|POLLERR)) { - mode += 'r'; - clear |= POLLIN; + var mode, clear int32 + if (ev.portev_events & (_POLLIN | _POLLHUP | _POLLERR)) != 0 { + mode += 'r' + clear |= _POLLIN } - if(ev->portev_events & (POLLOUT|POLLHUP|POLLERR)) { - mode += 'w'; - clear |= POLLOUT; + if (ev.portev_events & (_POLLOUT | _POLLHUP | _POLLERR)) != 0 { + mode += 'w' + clear |= _POLLOUT } // To effect edge-triggered events, we need to be sure to // update our association with whatever events were not @@ -248,17 +225,19 @@ retry: // for POLLIN|POLLOUT, and we get POLLIN, besides waking // the goroutine interested in POLLIN we have to not forget // about the one interested in POLLOUT. - if(clear != 0) { - runtime·netpolllock(pd); - runtime·netpollupdate(pd, 0, clear); - runtime·netpollunlock(pd); + if clear != 0 { + lock(&pd.lock) + netpollupdate(pd, 0, uint32(clear)) + unlock(&pd.lock) } - if(mode) - runtime·netpollready(&gp, pd, mode); + if mode != 0 { + netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode) + } } - if(block && gp == nil) - goto retry; - return gp; + if block && gp == nil { + goto retry + } + return gp } diff --git a/src/runtime/netpoll_windows.c b/src/runtime/netpoll_windows.c deleted file mode 100644 index 64da41ad9..000000000 --- a/src/runtime/netpoll_windows.c +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" - -#define DWORD_MAX 0xffffffff - -#pragma dynimport runtime·CreateIoCompletionPort CreateIoCompletionPort "kernel32.dll" -#pragma dynimport runtime·GetQueuedCompletionStatus GetQueuedCompletionStatus "kernel32.dll" -#pragma dynimport runtime·WSAGetOverlappedResult WSAGetOverlappedResult "ws2_32.dll" - -extern void *runtime·CreateIoCompletionPort; -extern void *runtime·GetQueuedCompletionStatus; -extern void *runtime·WSAGetOverlappedResult; - -#define INVALID_HANDLE_VALUE ((uintptr)-1) - -// net_op must be the same as beginning of net.operation. Keep these in sync. -typedef struct net_op net_op; -struct net_op -{ - // used by windows - Overlapped o; - // used by netpoll - PollDesc* pd; - int32 mode; - int32 errno; - uint32 qty; -}; - -typedef struct OverlappedEntry OverlappedEntry; -struct OverlappedEntry -{ - uintptr key; - net_op* op; // In reality it's Overlapped*, but we cast it to net_op* anyway. - uintptr internal; - uint32 qty; -}; - -static void handlecompletion(G **gpp, net_op *o, int32 errno, uint32 qty); - -static uintptr iocphandle = INVALID_HANDLE_VALUE; // completion port io handle - -void -runtime·netpollinit(void) -{ - iocphandle = (uintptr)runtime·stdcall4(runtime·CreateIoCompletionPort, INVALID_HANDLE_VALUE, 0, 0, DWORD_MAX); - if(iocphandle == 0) { - runtime·printf("netpoll: failed to create iocp handle (errno=%d)\n", runtime·getlasterror()); - runtime·throw("netpoll: failed to create iocp handle"); - } - return; -} - -int32 -runtime·netpollopen(uintptr fd, PollDesc *pd) -{ - USED(pd); - if(runtime·stdcall4(runtime·CreateIoCompletionPort, fd, iocphandle, 0, 0) == 0) - return -runtime·getlasterror(); - return 0; -} - -int32 -runtime·netpollclose(uintptr fd) -{ - // nothing to do - USED(fd); - return 0; -} - -void -runtime·netpollarm(PollDesc* pd, int32 mode) -{ - USED(pd, mode); - runtime·throw("unused"); -} - -// Polls for completed network IO. -// Returns list of goroutines that become runnable. -G* -runtime·netpoll(bool block) -{ - OverlappedEntry entries[64]; - uint32 wait, qty, key, flags, n, i; - int32 errno; - net_op *op; - G *gp; - - if(iocphandle == INVALID_HANDLE_VALUE) - return nil; - gp = nil; - wait = 0; - if(block) - wait = INFINITE; -retry: - if(runtime·GetQueuedCompletionStatusEx != nil) { - n = nelem(entries) / runtime·gomaxprocs; - if(n < 8) - n = 8; - if(block) - g->m->blocked = true; - if(runtime·stdcall6(runtime·GetQueuedCompletionStatusEx, iocphandle, (uintptr)entries, n, (uintptr)&n, wait, 0) == 0) { - g->m->blocked = false; - errno = runtime·getlasterror(); - if(!block && errno == WAIT_TIMEOUT) - return nil; - runtime·printf("netpoll: GetQueuedCompletionStatusEx failed (errno=%d)\n", errno); - runtime·throw("netpoll: GetQueuedCompletionStatusEx failed"); - } - g->m->blocked = false; - for(i = 0; i < n; i++) { - op = entries[i].op; - errno = 0; - qty = 0; - if(runtime·stdcall5(runtime·WSAGetOverlappedResult, runtime·netpollfd(op->pd), (uintptr)op, (uintptr)&qty, 0, (uintptr)&flags) == 0) - errno = runtime·getlasterror(); - handlecompletion(&gp, op, errno, qty); - } - } else { - op = nil; - errno = 0; - qty = 0; - if(block) - g->m->blocked = true; - if(runtime·stdcall5(runtime·GetQueuedCompletionStatus, iocphandle, (uintptr)&qty, (uintptr)&key, (uintptr)&op, wait) == 0) { - g->m->blocked = false; - errno = runtime·getlasterror(); - if(!block && errno == WAIT_TIMEOUT) - return nil; - if(op == nil) { - runtime·printf("netpoll: GetQueuedCompletionStatus failed (errno=%d)\n", errno); - runtime·throw("netpoll: GetQueuedCompletionStatus failed"); - } - // dequeued failed IO packet, so report that - } - g->m->blocked = false; - handlecompletion(&gp, op, errno, qty); - } - if(block && gp == nil) - goto retry; - return gp; -} - -static void -handlecompletion(G **gpp, net_op *op, int32 errno, uint32 qty) -{ - int32 mode; - - if(op == nil) - runtime·throw("netpoll: GetQueuedCompletionStatus returned op == nil"); - mode = op->mode; - if(mode != 'r' && mode != 'w') { - runtime·printf("netpoll: GetQueuedCompletionStatus returned invalid mode=%d\n", mode); - runtime·throw("netpoll: GetQueuedCompletionStatus returned invalid mode"); - } - op->errno = errno; - op->qty = qty; - runtime·netpollready(gpp, op->pd, mode); -} diff --git a/src/runtime/netpoll_windows.go b/src/runtime/netpoll_windows.go new file mode 100644 index 000000000..23d5dc00f --- /dev/null +++ b/src/runtime/netpoll_windows.go @@ -0,0 +1,156 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +const _DWORD_MAX = 0xffffffff + +//go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort "kernel32.dll" +//go:cgo_import_dynamic runtime._GetQueuedCompletionStatus GetQueuedCompletionStatus "kernel32.dll" +//go:cgo_import_dynamic runtime._WSAGetOverlappedResult WSAGetOverlappedResult "ws2_32.dll" + +var ( + _CreateIoCompletionPort, + _GetQueuedCompletionStatus, + _WSAGetOverlappedResult stdFunction +) + +const _INVALID_HANDLE_VALUE = ^uintptr(0) + +// net_op must be the same as beginning of net.operation. Keep these in sync. +type net_op struct { + // used by windows + o overlapped + // used by netpoll + pd *pollDesc + mode int32 + errno int32 + qty uint32 +} + +type overlappedEntry struct { + key uintptr + op *net_op // In reality it's *overlapped, but we cast it to *net_op anyway. + internal uintptr + qty uint32 +} + +var iocphandle uintptr = _INVALID_HANDLE_VALUE // completion port io handle + +func netpollinit() { + iocphandle = uintptr(stdcall4(_CreateIoCompletionPort, _INVALID_HANDLE_VALUE, 0, 0, _DWORD_MAX)) + if iocphandle == 0 { + println("netpoll: failed to create iocp handle (errno=", getlasterror(), ")") + gothrow("netpoll: failed to create iocp handle") + } +} + +func netpollopen(fd uintptr, pd *pollDesc) int32 { + if stdcall4(_CreateIoCompletionPort, fd, iocphandle, 0, 0) == 0 { + return -int32(getlasterror()) + } + return 0 +} + +func netpollclose(fd uintptr) int32 { + // nothing to do + return 0 +} + +func netpollarm(pd *pollDesc, mode int) { + gothrow("unused") +} + +// Polls for completed network IO. +// Returns list of goroutines that become runnable. +func netpoll(block bool) *g { + var entries [64]overlappedEntry + var wait, qty, key, flags, n, i uint32 + var errno int32 + var op *net_op + var gp *g + + mp := getg().m + + if iocphandle == _INVALID_HANDLE_VALUE { + return nil + } + gp = nil + wait = 0 + if block { + wait = _INFINITE + } +retry: + if _GetQueuedCompletionStatusEx != nil { + n = uint32(len(entries) / int(gomaxprocs)) + if n < 8 { + n = 8 + } + if block { + mp.blocked = true + } + if stdcall6(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 { + mp.blocked = false + errno = int32(getlasterror()) + if !block && errno == _WAIT_TIMEOUT { + return nil + } + println("netpoll: GetQueuedCompletionStatusEx failed (errno=", errno, ")") + gothrow("netpoll: GetQueuedCompletionStatusEx failed") + } + mp.blocked = false + for i = 0; i < n; i++ { + op = entries[i].op + errno = 0 + qty = 0 + if stdcall5(_WSAGetOverlappedResult, op.pd.fd, uintptr(unsafe.Pointer(op)), uintptr(unsafe.Pointer(&qty)), 0, uintptr(unsafe.Pointer(&flags))) == 0 { + errno = int32(getlasterror()) + } + handlecompletion(&gp, op, errno, qty) + } + } else { + op = nil + errno = 0 + qty = 0 + if block { + mp.blocked = true + } + if stdcall5(_GetQueuedCompletionStatus, iocphandle, uintptr(unsafe.Pointer(&qty)), uintptr(unsafe.Pointer(&key)), uintptr(unsafe.Pointer(&op)), uintptr(wait)) == 0 { + mp.blocked = false + errno = int32(getlasterror()) + if !block && errno == _WAIT_TIMEOUT { + return nil + } + if op == nil { + println("netpoll: GetQueuedCompletionStatus failed (errno=", errno, ")") + gothrow("netpoll: GetQueuedCompletionStatus failed") + } + // dequeued failed IO packet, so report that + } + mp.blocked = false + handlecompletion(&gp, op, errno, qty) + } + if block && gp == nil { + goto retry + } + return gp +} + +func handlecompletion(gpp **g, op *net_op, errno int32, qty uint32) { + if op == nil { + gothrow("netpoll: GetQueuedCompletionStatus returned op == nil") + } + mode := op.mode + if mode != 'r' && mode != 'w' { + println("netpoll: GetQueuedCompletionStatus returned invalid mode=", mode) + gothrow("netpoll: GetQueuedCompletionStatus returned invalid mode") + } + op.errno = errno + op.qty = qty + netpollready((**g)(noescape(unsafe.Pointer(gpp))), op.pd, mode) +} diff --git a/src/runtime/noasm_arm.go b/src/runtime/noasm.go index dd3ef8267..43c16860b 100644 --- a/src/runtime/noasm_arm.go +++ b/src/runtime/noasm.go @@ -5,6 +5,8 @@ // Routines that are implemented in assembly in asm_{amd64,386}.s // but are implemented in Go for arm. +// +build arm power64 power64le + package runtime func cmpstring(s1, s2 string) int { diff --git a/src/runtime/norace_test.go b/src/runtime/norace_test.go index 3b171877a..3681bf190 100644 --- a/src/runtime/norace_test.go +++ b/src/runtime/norace_test.go @@ -34,12 +34,12 @@ func benchmarkSyscall(b *testing.B, work, excess int) { b.RunParallel(func(pb *testing.PB) { foo := 42 for pb.Next() { - runtime.Entersyscall() + runtime.Entersyscall(0) for i := 0; i < work; i++ { foo *= 2 foo /= 2 } - runtime.Exitsyscall() + runtime.Exitsyscall(0) } _ = foo }) diff --git a/src/runtime/os1_darwin.go b/src/runtime/os1_darwin.go new file mode 100644 index 000000000..2fbf2cae0 --- /dev/null +++ b/src/runtime/os1_darwin.go @@ -0,0 +1,423 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//extern SigTabTT runtime·sigtab[]; + +var sigset_none = uint32(0) +var sigset_all = ^uint32(0) + +func unimplemented(name string) { + println(name, "not implemented") + *(*int)(unsafe.Pointer(uintptr(1231))) = 1231 +} + +//go:nosplit +func semawakeup(mp *m) { + mach_semrelease(uint32(mp.waitsema)) +} + +//go:nosplit +func semacreate() uintptr { + var x uintptr + systemstack(func() { + x = uintptr(mach_semcreate()) + }) + return x +} + +// BSD interface for threading. +func osinit() { + // bsdthread_register delayed until end of goenvs so that we + // can look at the environment first. + + // Use sysctl to fetch hw.ncpu. + mib := [2]uint32{6, 3} + out := uint32(0) + nout := unsafe.Sizeof(out) + ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret >= 0 { + ncpu = int32(out) + } +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() + + // Register our thread-creation callback (see sys_darwin_{amd64,386}.s) + // but only if we're not using cgo. If we are using cgo we need + // to let the C pthread library install its own thread-creation callback. + if !iscgo { + if bsdthread_register() != 0 { + if gogetenv("DYLD_INSERT_LIBRARIES") != "" { + gothrow("runtime: bsdthread_register error (unset DYLD_INSERT_LIBRARIES)") + } + gothrow("runtime: bsdthread_register error") + } + } +} + +func newosproc(mp *m, stk unsafe.Pointer) { + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, "/", int(mp.tls[0]), " ostk=", &mp, "\n") + } + + var oset uint32 + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + errno := bsdthread_create(stk, mp, mp.g0, funcPC(mstart)) + sigprocmask(_SIG_SETMASK, &oset, nil) + + if errno < 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -errno, ")\n") + gothrow("runtime.newosproc") + } +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) // OS X wants >= 8K + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + // Initialize signal handling. + _g_ := getg() + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +// Mach IPC, to get at semaphores +// Definitions are in /usr/include/mach on a Mac. + +func macherror(r int32, fn string) { + print("mach error ", fn, ": ", r, "\n") + gothrow("mach error") +} + +const _DebugMach = false + +var zerondr machndr + +func mach_msgh_bits(a, b uint32) uint32 { + return a | b<<8 +} + +func mach_msg(h *machheader, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32 { + // TODO: Loop on interrupt. + return mach_msg_trap(unsafe.Pointer(h), op, send_size, rcv_size, rcv_name, timeout, notify) +} + +// Mach RPC (MIG) +const ( + _MinMachMsg = 48 + _MachReply = 100 +) + +type codemsg struct { + h machheader + ndr machndr + code int32 +} + +func machcall(h *machheader, maxsize int32, rxsize int32) int32 { + _g_ := getg() + port := _g_.m.machport + if port == 0 { + port = mach_reply_port() + _g_.m.machport = port + } + + h.msgh_bits |= mach_msgh_bits(_MACH_MSG_TYPE_COPY_SEND, _MACH_MSG_TYPE_MAKE_SEND_ONCE) + h.msgh_local_port = port + h.msgh_reserved = 0 + id := h.msgh_id + + if _DebugMach { + p := (*[10000]unsafe.Pointer)(unsafe.Pointer(h)) + print("send:\t") + var i uint32 + for i = 0; i < h.msgh_size/uint32(unsafe.Sizeof(p[0])); i++ { + print(" ", p[i]) + if i%8 == 7 { + print("\n\t") + } + } + if i%8 != 0 { + print("\n") + } + } + ret := mach_msg(h, _MACH_SEND_MSG|_MACH_RCV_MSG, h.msgh_size, uint32(maxsize), port, 0, 0) + if ret != 0 { + if _DebugMach { + print("mach_msg error ", ret, "\n") + } + return ret + } + if _DebugMach { + p := (*[10000]unsafe.Pointer)(unsafe.Pointer(h)) + var i uint32 + for i = 0; i < h.msgh_size/uint32(unsafe.Sizeof(p[0])); i++ { + print(" ", p[i]) + if i%8 == 7 { + print("\n\t") + } + } + if i%8 != 0 { + print("\n") + } + } + if h.msgh_id != id+_MachReply { + if _DebugMach { + print("mach_msg _MachReply id mismatch ", h.msgh_id, " != ", id+_MachReply, "\n") + } + return -303 // MIG_REPLY_MISMATCH + } + // Look for a response giving the return value. + // Any call can send this back with an error, + // and some calls only have return values so they + // send it back on success too. I don't quite see how + // you know it's one of these and not the full response + // format, so just look if the message is right. + c := (*codemsg)(unsafe.Pointer(h)) + if uintptr(h.msgh_size) == unsafe.Sizeof(*c) && h.msgh_bits&_MACH_MSGH_BITS_COMPLEX == 0 { + if _DebugMach { + print("mig result ", c.code, "\n") + } + return c.code + } + if h.msgh_size != uint32(rxsize) { + if _DebugMach { + print("mach_msg _MachReply size mismatch ", h.msgh_size, " != ", rxsize, "\n") + } + return -307 // MIG_ARRAY_TOO_LARGE + } + return 0 +} + +// Semaphores! + +const ( + tmach_semcreate = 3418 + rmach_semcreate = tmach_semcreate + _MachReply + + tmach_semdestroy = 3419 + rmach_semdestroy = tmach_semdestroy + _MachReply + + _KERN_ABORTED = 14 + _KERN_OPERATION_TIMED_OUT = 49 +) + +type tmach_semcreatemsg struct { + h machheader + ndr machndr + policy int32 + value int32 +} + +type rmach_semcreatemsg struct { + h machheader + body machbody + semaphore machport +} + +type tmach_semdestroymsg struct { + h machheader + body machbody + semaphore machport +} + +func mach_semcreate() uint32 { + var m [256]uint8 + tx := (*tmach_semcreatemsg)(unsafe.Pointer(&m)) + rx := (*rmach_semcreatemsg)(unsafe.Pointer(&m)) + + tx.h.msgh_bits = 0 + tx.h.msgh_size = uint32(unsafe.Sizeof(*tx)) + tx.h.msgh_remote_port = mach_task_self() + tx.h.msgh_id = tmach_semcreate + tx.ndr = zerondr + + tx.policy = 0 // 0 = SYNC_POLICY_FIFO + tx.value = 0 + + for { + r := machcall(&tx.h, int32(unsafe.Sizeof(m)), int32(unsafe.Sizeof(*rx))) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_create") + } + if rx.body.msgh_descriptor_count != 1 { + unimplemented("mach_semcreate desc count") + } + return rx.semaphore.name +} + +func mach_semdestroy(sem uint32) { + var m [256]uint8 + tx := (*tmach_semdestroymsg)(unsafe.Pointer(&m)) + + tx.h.msgh_bits = _MACH_MSGH_BITS_COMPLEX + tx.h.msgh_size = uint32(unsafe.Sizeof(*tx)) + tx.h.msgh_remote_port = mach_task_self() + tx.h.msgh_id = tmach_semdestroy + tx.body.msgh_descriptor_count = 1 + tx.semaphore.name = sem + tx.semaphore.disposition = _MACH_MSG_TYPE_MOVE_SEND + tx.semaphore._type = 0 + + for { + r := machcall(&tx.h, int32(unsafe.Sizeof(m)), 0) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_destroy") + } +} + +// The other calls have simple system call traps in sys_darwin_{amd64,386}.s + +func mach_semaphore_wait(sema uint32) int32 +func mach_semaphore_timedwait(sema, sec, nsec uint32) int32 +func mach_semaphore_signal(sema uint32) int32 +func mach_semaphore_signal_all(sema uint32) int32 + +func semasleep1(ns int64) int32 { + _g_ := getg() + + if ns >= 0 { + var nsecs int32 + secs := timediv(ns, 1000000000, &nsecs) + r := mach_semaphore_timedwait(uint32(_g_.m.waitsema), uint32(secs), uint32(nsecs)) + if r == _KERN_ABORTED || r == _KERN_OPERATION_TIMED_OUT { + return -1 + } + if r != 0 { + macherror(r, "semaphore_wait") + } + return 0 + } + + for { + r := mach_semaphore_wait(uint32(_g_.m.waitsema)) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_wait") + } + return 0 +} + +//go:nosplit +func semasleep(ns int64) int32 { + var r int32 + systemstack(func() { + r = semasleep1(ns) + }) + return r +} + +//go:nosplit +func mach_semrelease(sem uint32) { + for { + r := mach_semaphore_signal(sem) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + + // mach_semrelease must be completely nosplit, + // because it is called from Go code. + // If we're going to die, start that process on the system stack + // to avoid a Go stack split. + systemstack(func() { macherror(r, "semaphore_signal") }) + } +} + +//go:nosplit +func osyield() { + usleep(1) +} + +func memlimit() uintptr { + // NOTE(rsc): Could use getrlimit here, + // like on FreeBSD or Linux, but Darwin doesn't enforce + // ulimit -v, so it's unclear why we'd try to stay within + // the limit. + return 0 +} + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = ^uint32(0) + sa.sa_tramp = unsafe.Pointer(funcPC(sigtramp)) // runtime·sigtramp's job is to call into real handler + *(*uintptr)(unsafe.Pointer(&sa.__sigaction_u)) = fn + sigaction(uint32(i), &sa, nil) +} + +func getsig(i int32) uintptr { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sigaction(uint32(i), nil, &sa) + return *(*uintptr)(unsafe.Pointer(&sa.__sigaction_u)) +} + +func signalstack(p *byte, n int32) { + var st stackt + st.ss_sp = p + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} diff --git a/src/runtime/os1_dragonfly.go b/src/runtime/os1_dragonfly.go new file mode 100644 index 000000000..82bb45b9b --- /dev/null +++ b/src/runtime/os1_dragonfly.go @@ -0,0 +1,220 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// From DragonFly's <sys/sysctl.h> +const ( + _CTL_HW = 6 + _HW_NCPU = 3 +) + +var sigset_none = sigset{} +var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} + +func getncpu() int32 { + mib := [2]uint32{_CTL_HW, _HW_NCPU} + out := uint32(0) + nout := unsafe.Sizeof(out) + ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret >= 0 { + return int32(out) + } + return 1 +} + +//go:nosplit +func futexsleep(addr *uint32, val uint32, ns int64) { + systemstack(func() { + futexsleep1(addr, val, ns) + }) +} + +func futexsleep1(addr *uint32, val uint32, ns int64) { + var timeout int32 + if ns >= 0 { + // The timeout is specified in microseconds - ensure that we + // do not end up dividing to zero, which would put us to sleep + // indefinitely... + timeout = timediv(ns, 1000, nil) + if timeout == 0 { + timeout = 1 + } + } + + // sys_umtx_sleep will return EWOULDBLOCK (EAGAIN) when the timeout + // expires or EBUSY if the mutex value does not match. + ret := sys_umtx_sleep(addr, int32(val), timeout) + if ret >= 0 || ret == -_EINTR || ret == -_EAGAIN || ret == -_EBUSY { + return + } + + print("umtx_sleep addr=", addr, " val=", val, " ret=", ret, "\n") + *(*int32)(unsafe.Pointer(uintptr(0x1005))) = 0x1005 +} + +//go:nosplit +func futexwakeup(addr *uint32, cnt uint32) { + ret := sys_umtx_wakeup(addr, int32(cnt)) + if ret >= 0 { + return + } + + systemstack(func() { + print("umtx_wake_addr=", addr, " ret=", ret, "\n") + *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 + }) +} + +func lwp_start(uintptr) + +func newosproc(mp *m, stk unsafe.Pointer) { + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " lwp_start=", funcPC(lwp_start), " id=", mp.id, "/", mp.tls[0], " ostk=", &mp, "\n") + } + + var oset sigset + sigprocmask(&sigset_all, &oset) + + params := lwpparams{ + start_func: funcPC(lwp_start), + arg: unsafe.Pointer(mp), + stack: uintptr(stk), + tid1: unsafe.Pointer(&mp.procid), + tid2: nil, + } + + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + + lwp_create(¶ms) + sigprocmask(&oset, nil) +} + +func osinit() { + ncpu = getncpu() +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/urandom\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + _g_ := getg() + + // m.procid is a uint64, but lwp_start writes an int32. Fix it up. + _g_.m.procid = uint64(*(*int32)(unsafe.Pointer(&_g_.m.procid))) + + // Initialize signal handling + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(&sigset_none, nil) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + /* + TODO: Convert to Go when something actually uses the result. + + Rlimit rl; + extern byte runtime·text[], runtime·end[]; + uintptr used; + + if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) + return 0; + if(rl.rlim_cur >= 0x7fffffff) + return 0; + + // Estimate our VM footprint excluding the heap. + // Not an exact science: use size of binary plus + // some room for thread stacks. + used = runtime·end - runtime·text + (64<<20); + if(used >= rl.rlim_cur) + return 0; + + // If there's not at least 16 MB left, we're probably + // not going to be able to do much. Treat as no limit. + rl.rlim_cur -= used; + if(rl.rlim_cur < (16<<20)) + return 0; + + return rl.rlim_cur - used; + */ + return 0 +} + +func sigtramp() + +type sigactiont struct { + sa_sigaction uintptr + sa_flags int32 + sa_mask sigset +} + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = sigset_all + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + sa.sa_sigaction = fn + sigaction(i, &sa, nil) +} + +func getsig(i int32) uintptr { + var sa sigactiont + sigaction(i, nil, &sa) + if sa.sa_sigaction == funcPC(sigtramp) { + return funcPC(sighandler) + } + return sa.sa_sigaction +} + +func signalstack(p *byte, n int32) { + var st sigaltstackt + st.ss_sp = uintptr(unsafe.Pointer(p)) + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(&sigset_none, nil) +} diff --git a/src/runtime/os1_freebsd.go b/src/runtime/os1_freebsd.go new file mode 100644 index 000000000..2cacfbae6 --- /dev/null +++ b/src/runtime/os1_freebsd.go @@ -0,0 +1,221 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// From FreeBSD's <sys/sysctl.h> +const ( + _CTL_HW = 6 + _HW_NCPU = 3 +) + +var sigset_none = sigset{} +var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} + +func getncpu() int32 { + mib := [2]uint32{_CTL_HW, _HW_NCPU} + out := uint32(0) + nout := unsafe.Sizeof(out) + ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret >= 0 { + return int32(out) + } + return 1 +} + +// FreeBSD's umtx_op syscall is effectively the same as Linux's futex, and +// thus the code is largely similar. See Linux implementation +// and lock_futex.c for comments. + +//go:nosplit +func futexsleep(addr *uint32, val uint32, ns int64) { + systemstack(func() { + futexsleep1(addr, val, ns) + }) +} + +func futexsleep1(addr *uint32, val uint32, ns int64) { + var tsp *timespec + if ns >= 0 { + var ts timespec + ts.tv_nsec = 0 + ts.set_sec(int64(timediv(ns, 1000000000, (*int32)(unsafe.Pointer(&ts.tv_nsec))))) + tsp = &ts + } + ret := sys_umtx_op(addr, _UMTX_OP_WAIT_UINT_PRIVATE, val, nil, tsp) + if ret >= 0 || ret == -_EINTR { + return + } + print("umtx_wait addr=", addr, " val=", val, " ret=", ret, "\n") + *(*int32)(unsafe.Pointer(uintptr(0x1005))) = 0x1005 +} + +//go:nosplit +func futexwakeup(addr *uint32, cnt uint32) { + ret := sys_umtx_op(addr, _UMTX_OP_WAKE_PRIVATE, cnt, nil, nil) + if ret >= 0 { + return + } + + systemstack(func() { + print("umtx_wake_addr=", addr, " ret=", ret, "\n") + }) +} + +func thr_start() + +func newosproc(mp *m, stk unsafe.Pointer) { + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " thr_start=", funcPC(thr_start), " id=", mp.id, "/", mp.tls[0], " ostk=", &mp, "\n") + } + + // NOTE(rsc): This code is confused. stackbase is the top of the stack + // and is equal to stk. However, it's working, so I'm not changing it. + param := thrparam{ + start_func: funcPC(thr_start), + arg: unsafe.Pointer(mp), + stack_base: mp.g0.stack.hi, + stack_size: uintptr(stk) - mp.g0.stack.hi, + child_tid: unsafe.Pointer(&mp.procid), + parent_tid: nil, + tls_base: unsafe.Pointer(&mp.tls[0]), + tls_size: unsafe.Sizeof(mp.tls), + } + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + + var oset sigset + sigprocmask(&sigset_all, &oset) + thr_new(¶m, int32(unsafe.Sizeof(param))) + sigprocmask(&oset, nil) +} + +func osinit() { + ncpu = getncpu() +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + _g_ := getg() + + // m.procid is a uint64, but thr_new writes a uint32 on 32-bit systems. + // Fix it up. (Only matters on big-endian, but be clean anyway.) + if ptrSize == 4 { + _g_.m.procid = uint64(*(*uint32)(unsafe.Pointer(&_g_.m.procid))) + } + + // Initialize signal handling. + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(&sigset_none, nil) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + /* + TODO: Convert to Go when something actually uses the result. + Rlimit rl; + extern byte runtime·text[], runtime·end[]; + uintptr used; + + if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) + return 0; + if(rl.rlim_cur >= 0x7fffffff) + return 0; + + // Estimate our VM footprint excluding the heap. + // Not an exact science: use size of binary plus + // some room for thread stacks. + used = runtime·end - runtime·text + (64<<20); + if(used >= rl.rlim_cur) + return 0; + + // If there's not at least 16 MB left, we're probably + // not going to be able to do much. Treat as no limit. + rl.rlim_cur -= used; + if(rl.rlim_cur < (16<<20)) + return 0; + + return rl.rlim_cur - used; + */ + + return 0 +} + +func sigtramp() + +type sigactiont struct { + sa_handler uintptr + sa_flags int32 + sa_mask sigset +} + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = sigset_all + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + sa.sa_handler = fn + sigaction(i, &sa, nil) +} +func getsig(i int32) uintptr { + var sa sigactiont + sigaction(i, nil, &sa) + if sa.sa_handler == funcPC(sigtramp) { + return funcPC(sighandler) + } + return sa.sa_handler +} + +func signalstack(p *byte, n int32) { + var st stackt + st.ss_sp = uintptr(unsafe.Pointer(p)) + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(&sigset_none, nil) +} diff --git a/src/runtime/os1_linux.go b/src/runtime/os1_linux.go new file mode 100644 index 000000000..67fa6391e --- /dev/null +++ b/src/runtime/os1_linux.go @@ -0,0 +1,287 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +var sigset_none sigset +var sigset_all sigset = sigset{^uint32(0), ^uint32(0)} + +// Linux futex. +// +// futexsleep(uint32 *addr, uint32 val) +// futexwakeup(uint32 *addr) +// +// Futexsleep atomically checks if *addr == val and if so, sleeps on addr. +// Futexwakeup wakes up threads sleeping on addr. +// Futexsleep is allowed to wake up spuriously. + +const ( + _FUTEX_WAIT = 0 + _FUTEX_WAKE = 1 +) + +// Atomically, +// if(*addr == val) sleep +// Might be woken up spuriously; that's allowed. +// Don't sleep longer than ns; ns < 0 means forever. +//go:nosplit +func futexsleep(addr *uint32, val uint32, ns int64) { + var ts timespec + + // Some Linux kernels have a bug where futex of + // FUTEX_WAIT returns an internal error code + // as an errno. Libpthread ignores the return value + // here, and so can we: as it says a few lines up, + // spurious wakeups are allowed. + if ns < 0 { + futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, nil, nil, 0) + return + } + + // It's difficult to live within the no-split stack limits here. + // On ARM and 386, a 64-bit divide invokes a general software routine + // that needs more stack than we can afford. So we use timediv instead. + // But on real 64-bit systems, where words are larger but the stack limit + // is not, even timediv is too heavy, and we really need to use just an + // ordinary machine instruction. + if ptrSize == 8 { + ts.set_sec(ns / 1000000000) + ts.set_nsec(int32(ns % 1000000000)) + } else { + ts.tv_nsec = 0 + ts.set_sec(int64(timediv(ns, 1000000000, (*int32)(unsafe.Pointer(&ts.tv_nsec))))) + } + futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, unsafe.Pointer(&ts), nil, 0) +} + +// If any procs are sleeping on addr, wake up at most cnt. +//go:nosplit +func futexwakeup(addr *uint32, cnt uint32) { + ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE, cnt, nil, nil, 0) + if ret >= 0 { + return + } + + // I don't know that futex wakeup can return + // EAGAIN or EINTR, but if it does, it would be + // safe to loop and call futex again. + systemstack(func() { + print("futexwakeup addr=", addr, " returned ", ret, "\n") + }) + + *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 +} + +func getproccount() int32 { + var buf [16]uintptr + r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) + n := int32(0) + for _, v := range buf[:r/ptrSize] { + for i := 0; i < 64; i++ { + n += int32(v & 1) + v >>= 1 + } + } + if n == 0 { + n = 1 + } + return n +} + +// Clone, the Linux rfork. +const ( + _CLONE_VM = 0x100 + _CLONE_FS = 0x200 + _CLONE_FILES = 0x400 + _CLONE_SIGHAND = 0x800 + _CLONE_PTRACE = 0x2000 + _CLONE_VFORK = 0x4000 + _CLONE_PARENT = 0x8000 + _CLONE_THREAD = 0x10000 + _CLONE_NEWNS = 0x20000 + _CLONE_SYSVSEM = 0x40000 + _CLONE_SETTLS = 0x80000 + _CLONE_PARENT_SETTID = 0x100000 + _CLONE_CHILD_CLEARTID = 0x200000 + _CLONE_UNTRACED = 0x800000 + _CLONE_CHILD_SETTID = 0x1000000 + _CLONE_STOPPED = 0x2000000 + _CLONE_NEWUTS = 0x4000000 + _CLONE_NEWIPC = 0x8000000 +) + +func newosproc(mp *m, stk unsafe.Pointer) { + /* + * note: strace gets confused if we use CLONE_PTRACE here. + */ + var flags int32 = _CLONE_VM | /* share memory */ + _CLONE_FS | /* share cwd, etc */ + _CLONE_FILES | /* share fd table */ + _CLONE_SIGHAND | /* share sig handler table */ + _CLONE_THREAD /* revisit - okay for now */ + + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", funcPC(clone), " id=", mp.id, "/", mp.tls[0], " ostk=", &mp, "\n") + } + + // Disable signals during clone, so that the new thread starts + // with signals disabled. It will enable them in minit. + var oset sigset + rtsigprocmask(_SIG_SETMASK, &sigset_all, &oset, int32(unsafe.Sizeof(oset))) + ret := clone(flags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(funcPC(mstart))) + rtsigprocmask(_SIG_SETMASK, &oset, nil, int32(unsafe.Sizeof(oset))) + + if ret < 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") + gothrow("newosproc") + } +} + +func osinit() { + ncpu = getproccount() +} + +// Random bytes initialized at startup. These come +// from the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.c). +// byte* runtime·startup_random_data; +// uint32 runtime·startup_random_data_len; + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + if startup_random_data != nil { + *rnd = unsafe.Pointer(startup_random_data) + *rnd_len = int32(startup_random_data_len) + return + } + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) // Linux wants >= 2K + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + // Initialize signal handling. + _g_ := getg() + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + rtsigprocmask(_SIG_SETMASK, &sigset_none, nil, int32(unsafe.Sizeof(sigset_none))) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + /* + TODO: Convert to Go when something actually uses the result. + + Rlimit rl; + extern byte runtime·text[], runtime·end[]; + uintptr used; + + if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) + return 0; + if(rl.rlim_cur >= 0x7fffffff) + return 0; + + // Estimate our VM footprint excluding the heap. + // Not an exact science: use size of binary plus + // some room for thread stacks. + used = runtime·end - runtime·text + (64<<20); + if(used >= rl.rlim_cur) + return 0; + + // If there's not at least 16 MB left, we're probably + // not going to be able to do much. Treat as no limit. + rl.rlim_cur -= used; + if(rl.rlim_cur < (16<<20)) + return 0; + + return rl.rlim_cur - used; + */ + + return 0 +} + +//#ifdef GOARCH_386 +//#define sa_handler k_sa_handler +//#endif + +func sigreturn() +func sigtramp() + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = ^uint64(0) + // Although Linux manpage says "sa_restorer element is obsolete and + // should not be used". x86_64 kernel requires it. Only use it on + // x86. + if GOARCH == "386" || GOARCH == "amd64" { + sa.sa_restorer = funcPC(sigreturn) + } + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + sa.sa_handler = fn + if rt_sigaction(uintptr(i), &sa, nil, unsafe.Sizeof(sa.sa_mask)) != 0 { + gothrow("rt_sigaction failure") + } +} + +func getsig(i int32) uintptr { + var sa sigactiont + + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + if rt_sigaction(uintptr(i), nil, &sa, unsafe.Sizeof(sa.sa_mask)) != 0 { + gothrow("rt_sigaction read failure") + } + if sa.sa_handler == funcPC(sigtramp) { + return funcPC(sighandler) + } + return sa.sa_handler +} + +func signalstack(p *byte, n int32) { + var st sigaltstackt + st.ss_sp = p + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + rtsigprocmask(_SIG_SETMASK, &sigset_none, nil, int32(unsafe.Sizeof(sigset_none))) +} diff --git a/src/runtime/os1_openbsd.go b/src/runtime/os1_openbsd.go new file mode 100644 index 000000000..d5ffe10a8 --- /dev/null +++ b/src/runtime/os1_openbsd.go @@ -0,0 +1,235 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + ESRCH = 3 + EAGAIN = 35 + EWOULDBLOCK = EAGAIN + ENOTSUP = 91 + + // From OpenBSD's sys/time.h + CLOCK_REALTIME = 0 + CLOCK_VIRTUAL = 1 + CLOCK_PROF = 2 + CLOCK_MONOTONIC = 3 +) + +var sigset_none = uint32(0) +var sigset_all = ^sigset_none + +// From OpenBSD's <sys/sysctl.h> +const ( + CTL_HW = 6 + HW_NCPU = 3 +) + +func getncpu() int32 { + mib := [2]uint32{CTL_HW, HW_NCPU} + out := uint32(0) + nout := unsafe.Sizeof(out) + + // Fetch hw.ncpu via sysctl. + ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret >= 0 { + return int32(out) + } + return 1 +} + +//go:nosplit +func semacreate() uintptr { + return 1 +} + +//go:nosplit +func semasleep(ns int64) int32 { + _g_ := getg() + + // Compute sleep deadline. + var tsp *timespec + if ns >= 0 { + var ts timespec + var nsec int32 + ns += nanotime() + ts.set_sec(int64(timediv(ns, 1000000000, &nsec))) + ts.set_nsec(nsec) + tsp = &ts + } + + for { + // spin-mutex lock + for { + if xchg(&_g_.m.waitsemalock, 1) == 0 { + break + } + osyield() + } + + if _g_.m.waitsemacount != 0 { + // semaphore is available. + _g_.m.waitsemacount-- + // spin-mutex unlock + atomicstore(&_g_.m.waitsemalock, 0) + return 0 // semaphore acquired + } + + // sleep until semaphore != 0 or timeout. + // thrsleep unlocks m.waitsemalock. + ret := thrsleep((uintptr)(unsafe.Pointer(&_g_.m.waitsemacount)), CLOCK_MONOTONIC, tsp, (uintptr)(unsafe.Pointer(&_g_.m.waitsemalock)), (*int32)(unsafe.Pointer(&_g_.m.waitsemacount))) + if ret == EWOULDBLOCK { + return -1 + } + } +} + +//go:nosplit +func semawakeup(mp *m) { + // spin-mutex lock + for { + if xchg(&mp.waitsemalock, 1) == 0 { + break + } + osyield() + } + mp.waitsemacount++ + ret := thrwakeup(uintptr(unsafe.Pointer(&mp.waitsemacount)), 1) + if ret != 0 && ret != ESRCH { + // semawakeup can be called on signal stack. + systemstack(func() { + print("thrwakeup addr=", &mp.waitsemacount, " sem=", mp.waitsemacount, " ret=", ret, "\n") + }) + } + // spin-mutex unlock + atomicstore(&mp.waitsemalock, 0) +} + +func newosproc(mp *m, stk unsafe.Pointer) { + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, "/", int32(mp.tls[0]), " ostk=", &mp, "\n") + } + + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + + param := tforkt{ + tf_tcb: unsafe.Pointer(&mp.tls[0]), + tf_tid: (*int32)(unsafe.Pointer(&mp.procid)), + tf_stack: uintptr(stk), + } + + oset := sigprocmask(_SIG_SETMASK, sigset_all) + ret := tfork(¶m, unsafe.Sizeof(param), mp, mp.g0, funcPC(mstart)) + sigprocmask(_SIG_SETMASK, oset) + + if ret < 0 { + print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", -ret, ")\n") + if ret == -ENOTSUP { + print("runtime: is kern.rthreads disabled?\n") + } + gothrow("runtime.newosproc") + } +} + +func osinit() { + ncpu = getncpu() +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/urandom\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + _g_ := getg() + + // m.procid is a uint64, but tfork writes an int32. Fix it up. + _g_.m.procid = uint64(*(*int32)(unsafe.Pointer(&_g_.m.procid))) + + // Initialize signal handling + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(_SIG_SETMASK, sigset_none) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + return 0 +} + +func sigtramp() + +type sigactiont struct { + sa_sigaction uintptr + sa_mask uint32 + sa_flags int32 +} + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = sigset_all + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + sa.sa_sigaction = fn + sigaction(i, &sa, nil) +} + +func getsig(i int32) uintptr { + var sa sigactiont + sigaction(i, nil, &sa) + if sa.sa_sigaction == funcPC(sigtramp) { + return funcPC(sighandler) + } + return sa.sa_sigaction +} + +func signalstack(p *byte, n int32) { + var st stackt + + st.ss_sp = uintptr(unsafe.Pointer(p)) + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(_SIG_SETMASK, sigset_none) +} diff --git a/src/runtime/os1_windows.go b/src/runtime/os1_windows.go new file mode 100644 index 000000000..abd2297a3 --- /dev/null +++ b/src/runtime/os1_windows.go @@ -0,0 +1,564 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +//go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler "kernel32.dll" +//go:cgo_import_dynamic runtime._CloseHandle CloseHandle "kernel32.dll" +//go:cgo_import_dynamic runtime._CreateEventA CreateEventA "kernel32.dll" +//go:cgo_import_dynamic runtime._CreateThread CreateThread "kernel32.dll" +//go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA "kernel32.dll" +//go:cgo_import_dynamic runtime._CryptAcquireContextW CryptAcquireContextW "advapi32.dll" +//go:cgo_import_dynamic runtime._CryptGenRandom CryptGenRandom "advapi32.dll" +//go:cgo_import_dynamic runtime._CryptReleaseContext CryptReleaseContext "advapi32.dll" +//go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle "kernel32.dll" +//go:cgo_import_dynamic runtime._ExitProcess ExitProcess "kernel32.dll" +//go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW "kernel32.dll" +//go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW "kernel32.dll" +//go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress "kernel32.dll" +//go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle "kernel32.dll" +//go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo "kernel32.dll" +//go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext "kernel32.dll" +//go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW "kernel32.dll" +//go:cgo_import_dynamic runtime._LoadLibraryA LoadLibraryA "kernel32.dll" +//go:cgo_import_dynamic runtime._NtWaitForSingleObject NtWaitForSingleObject "ntdll.dll" +//go:cgo_import_dynamic runtime._ResumeThread ResumeThread "kernel32.dll" +//go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler "kernel32.dll" +//go:cgo_import_dynamic runtime._SetEvent SetEvent "kernel32.dll" +//go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost "kernel32.dll" +//go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority "kernel32.dll" +//go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter "kernel32.dll" +//go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer "kernel32.dll" +//go:cgo_import_dynamic runtime._Sleep Sleep "kernel32.dll" +//go:cgo_import_dynamic runtime._SuspendThread SuspendThread "kernel32.dll" +//go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject "kernel32.dll" +//go:cgo_import_dynamic runtime._WriteFile WriteFile "kernel32.dll" +//go:cgo_import_dynamic runtime._timeBeginPeriod timeBeginPeriod "winmm.dll" + +var ( + _AddVectoredExceptionHandler, + _CloseHandle, + _CreateEventA, + _CreateThread, + _CreateWaitableTimerA, + _CryptAcquireContextW, + _CryptGenRandom, + _CryptReleaseContext, + _DuplicateHandle, + _ExitProcess, + _FreeEnvironmentStringsW, + _GetEnvironmentStringsW, + _GetProcAddress, + _GetStdHandle, + _GetSystemInfo, + _GetThreadContext, + _LoadLibraryW, + _LoadLibraryA, + _NtWaitForSingleObject, + _ResumeThread, + _SetConsoleCtrlHandler, + _SetEvent, + _SetProcessPriorityBoost, + _SetThreadPriority, + _SetUnhandledExceptionFilter, + _SetWaitableTimer, + _Sleep, + _SuspendThread, + _WaitForSingleObject, + _WriteFile, + _timeBeginPeriod stdFunction +) + +var _GetQueuedCompletionStatusEx stdFunction + +// in sys_windows_386.s and sys_windows_amd64.s +func externalthreadhandler() +func exceptiontramp() +func firstcontinuetramp() +func lastcontinuetramp() + +//go:nosplit +func getLoadLibrary() uintptr { + return uintptr(unsafe.Pointer(_LoadLibraryW)) +} + +//go:nosplit +func getGetProcAddress() uintptr { + return uintptr(unsafe.Pointer(_GetProcAddress)) +} + +func getproccount() int32 { + var info systeminfo + stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) + return int32(info.dwnumberofprocessors) +} + +const ( + currentProcess = ^uintptr(0) // -1 = current process + currentThread = ^uintptr(1) // -2 = current thread +) + +var ( + kernel32Name = []byte("kernel32.dll\x00") + addVectoredContinueHandlerName = []byte("AddVectoredContinueHandler\x00") + getQueuedCompletionStatusExName = []byte("GetQueuedCompletionStatusEx\x00") +) + +func osinit() { + setBadSignalMsg() + + kernel32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&kernel32Name[0]))) + + externalthreadhandlerp = funcPC(externalthreadhandler) + + stdcall2(_AddVectoredExceptionHandler, 1, funcPC(exceptiontramp)) + addVectoredContinueHandler := uintptr(0) + if kernel32 != 0 { + addVectoredContinueHandler = stdcall2(_GetProcAddress, kernel32, uintptr(unsafe.Pointer(&addVectoredContinueHandlerName[0]))) + } + if addVectoredContinueHandler == 0 || unsafe.Sizeof(&kernel32) == 4 { + // use SetUnhandledExceptionFilter for windows-386 or + // if VectoredContinueHandler is unavailable. + // note: SetUnhandledExceptionFilter handler won't be called, if debugging. + stdcall1(_SetUnhandledExceptionFilter, funcPC(lastcontinuetramp)) + } else { + stdcall2(stdFunction(unsafe.Pointer(addVectoredContinueHandler)), 1, funcPC(firstcontinuetramp)) + stdcall2(stdFunction(unsafe.Pointer(addVectoredContinueHandler)), 0, funcPC(lastcontinuetramp)) + } + + stdcall2(_SetConsoleCtrlHandler, funcPC(ctrlhandler), 1) + + stdcall1(_timeBeginPeriod, 1) + + ncpu = getproccount() + + // Windows dynamic priority boosting assumes that a process has different types + // of dedicated threads -- GUI, IO, computational, etc. Go processes use + // equivalent threads that all do a mix of GUI, IO, computations, etc. + // In such context dynamic priority boosting does nothing but harm, so we turn it off. + stdcall2(_SetProcessPriorityBoost, currentProcess, 1) + + if kernel32 != 0 { + _GetQueuedCompletionStatusEx = stdFunction(unsafe.Pointer(stdcall2(_GetProcAddress, kernel32, uintptr(unsafe.Pointer(&getQueuedCompletionStatusExName[0]))))) + } +} + +var random_data [_HashRandomBytes]byte + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + const ( + prov_rsa_full = 1 + crypt_verifycontext = 0xF0000000 + ) + var handle uintptr + *rnd = nil + *rnd_len = 0 + if stdcall5(_CryptAcquireContextW, uintptr(unsafe.Pointer(&handle)), 0, 0, prov_rsa_full, crypt_verifycontext) != 0 { + if stdcall3(_CryptGenRandom, handle, _HashRandomBytes, uintptr(unsafe.Pointer(&random_data[0]))) != 0 { + *rnd = unsafe.Pointer(&random_data[0]) + *rnd_len = _HashRandomBytes + } + stdcall2(_CryptReleaseContext, handle, 0) + } +} + +func goenvs() { + var p *uint16 + + env := (*uint16)(unsafe.Pointer(stdcall0(_GetEnvironmentStringsW))) + + n := 0 + for p = env; *p != 0; n++ { + p = (*uint16)(add(unsafe.Pointer(p), uintptr(findnullw(p)+1))) + } + + envs = makeStringSlice(int(n)) + + p = env + for i := 0; i < n; i++ { + envs[i] = gostringw(p) + p = (*uint16)(add(unsafe.Pointer(p), uintptr(findnullw(p)+1))) + } + + stdcall1(_FreeEnvironmentStringsW, uintptr(unsafe.Pointer(env))) +} + +//go:nosplit +func exit(code int32) { + stdcall1(_ExitProcess, uintptr(code)) +} + +//go:nosplit +func write(fd uintptr, buf unsafe.Pointer, n int32) int32 { + const ( + _STD_OUTPUT_HANDLE = ^uintptr(10) // -11 + _STD_ERROR_HANDLE = ^uintptr(11) // -12 + ) + var handle uintptr + switch fd { + case 1: + handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE) + case 2: + handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE) + default: + // assume fd is real windows handle. + handle = fd + } + var written uint32 + stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) + return int32(written) +} + +//go:nosplit +func semasleep(ns int64) int32 { + // store ms in ns to save stack space + if ns < 0 { + ns = _INFINITE + } else { + ns = int64(timediv(ns, 1000000, nil)) + if ns == 0 { + ns = 1 + } + } + if stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(ns)) != 0 { + return -1 // timeout + } + return 0 +} + +//go:nosplit +func semawakeup(mp *m) { + stdcall1(_SetEvent, mp.waitsema) +} + +//go:nosplit +func semacreate() uintptr { + return stdcall4(_CreateEventA, 0, 0, 0, 0) +} + +func newosproc(mp *m, stk unsafe.Pointer) { + const _STACK_SIZE_PARAM_IS_A_RESERVATION = 0x00010000 + thandle := stdcall6(_CreateThread, 0, 0x20000, + funcPC(tstart_stdcall), uintptr(unsafe.Pointer(mp)), + _STACK_SIZE_PARAM_IS_A_RESERVATION, 0) + if thandle == 0 { + println("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")") + gothrow("runtime.newosproc") + } +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + var thandle uintptr + stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) + atomicstoreuintptr(&getg().m.thread, thandle) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + tp := &getg().m.thread + stdcall1(_CloseHandle, *tp) + *tp = 0 +} + +// Described in http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/ +type _KSYSTEM_TIME struct { + LowPart uint32 + High1Time int32 + High2Time int32 +} + +const ( + _INTERRUPT_TIME = 0x7ffe0008 + _SYSTEM_TIME = 0x7ffe0014 +) + +//go:nosplit +func systime(addr uintptr) int64 { + timeaddr := (*_KSYSTEM_TIME)(unsafe.Pointer(addr)) + + var t _KSYSTEM_TIME + for i := 1; i < 10000; i++ { + // these fields must be read in that order (see URL above) + t.High1Time = timeaddr.High1Time + t.LowPart = timeaddr.LowPart + t.High2Time = timeaddr.High2Time + if t.High1Time == t.High2Time { + return int64(t.High1Time)<<32 | int64(t.LowPart) + } + if (i % 100) == 0 { + osyield() + } + } + systemstack(func() { + gothrow("interrupt/system time is changing too fast") + }) + return 0 +} + +//go:nosplit +func unixnano() int64 { + return (systime(_SYSTEM_TIME) - 116444736000000000) * 100 +} + +//go:nosplit +func nanotime() int64 { + return systime(_INTERRUPT_TIME) * 100 +} + +// Calling stdcall on os stack. +//go:nosplit +func stdcall(fn stdFunction) uintptr { + gp := getg() + mp := gp.m + mp.libcall.fn = uintptr(unsafe.Pointer(fn)) + + if mp.profilehz != 0 { + // leave pc/sp for cpu profiler + mp.libcallg = gp + mp.libcallpc = getcallerpc(unsafe.Pointer(&fn)) + // sp must be the last, because once async cpu profiler finds + // all three values to be non-zero, it will use them + mp.libcallsp = getcallersp(unsafe.Pointer(&fn)) + } + asmcgocall(unsafe.Pointer(funcPC(asmstdcall)), unsafe.Pointer(&mp.libcall)) + mp.libcallsp = 0 + return mp.libcall.r1 +} + +//go:nosplit +func stdcall0(fn stdFunction) uintptr { + mp := getg().m + mp.libcall.n = 0 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&fn))) // it's unused but must be non-nil, otherwise crashes + return stdcall(fn) +} + +//go:nosplit +func stdcall1(fn stdFunction, a0 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 1 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 2 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 3 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 4 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 5 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 6 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +//go:nosplit +func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { + mp := getg().m + mp.libcall.n = 7 + mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + return stdcall(fn) +} + +// in sys_windows_386.s and sys_windows_amd64.s +func usleep1(usec uint32) + +//go:nosplit +func osyield() { + usleep1(1) +} + +//go:nosplit +func usleep(us uint32) { + // Have 1us units; want 100ns units. + usleep1(10 * us) +} + +func issigpanic(code uint32) uint32 { + switch code { + default: + return 0 + case _EXCEPTION_ACCESS_VIOLATION: + case _EXCEPTION_INT_DIVIDE_BY_ZERO: + case _EXCEPTION_INT_OVERFLOW: + case _EXCEPTION_FLT_DENORMAL_OPERAND: + case _EXCEPTION_FLT_DIVIDE_BY_ZERO: + case _EXCEPTION_FLT_INEXACT_RESULT: + case _EXCEPTION_FLT_OVERFLOW: + case _EXCEPTION_FLT_UNDERFLOW: + case _EXCEPTION_BREAKPOINT: + } + return 1 +} + +func initsig() { + /* + // TODO(brainman): I don't think we need that bit of code + // following line keeps these functions alive at link stage + // if there's a better way please write it here + void *e = runtime·exceptiontramp; + void *f = runtime·firstcontinuetramp; + void *l = runtime·lastcontinuetramp; + USED(e); + USED(f); + USED(l); + */ +} + +func ctrlhandler1(_type uint32) uint32 { + var s uint32 + + switch _type { + case _CTRL_C_EVENT, _CTRL_BREAK_EVENT: + s = _SIGINT + default: + return 0 + } + + if sigsend(s) { + return 1 + } + exit(2) // SIGINT, SIGTERM, etc + return 0 +} + +// in sys_windows_386.s and sys_windows_amd64.s +func profileloop() + +var profiletimer uintptr + +func profilem(mp *m) { + var r *context + rbuf := make([]byte, unsafe.Sizeof(*r)+15) + + tls := &mp.tls[0] + if mp == &m0 { + tls = &tls0[0] + } + gp := *((**g)(unsafe.Pointer(tls))) + + // align Context to 16 bytes + r = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&rbuf[15]))) &^ 15)) + r.contextflags = _CONTEXT_CONTROL + stdcall2(_GetThreadContext, mp.thread, uintptr(unsafe.Pointer(r))) + dosigprof(r, gp, mp) +} + +func profileloop1() { + stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST) + + for { + stdcall2(_WaitForSingleObject, profiletimer, _INFINITE) + first := (*m)(atomicloadp(unsafe.Pointer(&allm))) + for mp := first; mp != nil; mp = mp.alllink { + thread := atomicloaduintptr(&mp.thread) + // Do not profile threads blocked on Notes, + // this includes idle worker threads, + // idle timer thread, idle heap scavenger, etc. + if thread == 0 || mp.profilehz == 0 || mp.blocked { + continue + } + stdcall1(_SuspendThread, thread) + if mp.profilehz != 0 && !mp.blocked { + profilem(mp) + } + stdcall1(_ResumeThread, thread) + } + } +} + +var cpuprofilerlock mutex + +func resetcpuprofiler(hz int32) { + lock(&cpuprofilerlock) + if profiletimer == 0 { + timer := stdcall3(_CreateWaitableTimerA, 0, 0, 0) + atomicstoreuintptr(&profiletimer, timer) + thread := stdcall6(_CreateThread, 0, 0, funcPC(profileloop), 0, 0, 0) + stdcall2(_SetThreadPriority, thread, _THREAD_PRIORITY_HIGHEST) + stdcall1(_CloseHandle, thread) + } + unlock(&cpuprofilerlock) + + ms := int32(0) + due := ^int64(^uint64(1 << 63)) + if hz > 0 { + ms = 1000 / hz + if ms == 0 { + ms = 1 + } + due = int64(ms) * -10000 + } + stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) + atomicstore((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) +} + +func memlimit() uintptr { + return 0 +} + +var ( + badsignalmsg [100]byte + badsignallen int32 +) + +func setBadSignalMsg() { + const msg = "runtime: signal received on thread not created by Go.\n" + for i, c := range msg { + badsignalmsg[i] = byte(c) + badsignallen++ + } +} + +func crash() { + // TODO: This routine should do whatever is needed + // to make the Windows program abort/crash as it + // would if Go was not intercepting signals. + // On Unix the routine would remove the custom signal + // handler and then raise a signal (like SIGABRT). + // Something like that should happen here. + // It's okay to leave this empty for now: if crash returns + // the ordinary exit-after-panic happens. +} diff --git a/src/runtime/os1_windows_386.go b/src/runtime/os1_windows_386.go new file mode 100644 index 000000000..0afef9156 --- /dev/null +++ b/src/runtime/os1_windows_386.go @@ -0,0 +1,118 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +var text struct{} + +func dumpregs(r *context) { + print("eax ", hex(r.eax), "\n") + print("ebx ", hex(r.ebx), "\n") + print("ecx ", hex(r.ecx), "\n") + print("edx ", hex(r.edx), "\n") + print("edi ", hex(r.edi), "\n") + print("esi ", hex(r.esi), "\n") + print("ebp ", hex(r.ebp), "\n") + print("esp ", hex(r.esp), "\n") + print("eip ", hex(r.eip), "\n") + print("eflags ", hex(r.eflags), "\n") + print("cs ", hex(r.segcs), "\n") + print("fs ", hex(r.segfs), "\n") + print("gs ", hex(r.seggs), "\n") +} + +func isgoexception(info *exceptionrecord, r *context) bool { + // Only handle exception if executing instructions in Go binary + // (not Windows library code). + if r.eip < uint32(uintptr(unsafe.Pointer(&text))) || uint32(uintptr(unsafe.Pointer(&etext))) < r.eip { + return false + } + + if issigpanic(info.exceptioncode) == 0 { + return false + } + + return true +} + +// Called by sigtramp from Windows VEH handler. +// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION) +// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH). +func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { + if !isgoexception(info, r) { + return _EXCEPTION_CONTINUE_SEARCH + } + + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = info.exceptioncode + gp.sigcode0 = uintptr(info.exceptioninformation[0]) + gp.sigcode1 = uintptr(info.exceptioninformation[1]) + gp.sigpc = uintptr(r.eip) + + // Only push runtime·sigpanic if r->eip != 0. + // If r->eip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime·sigpanic instead. + // (Otherwise the trace will end at runtime·sigpanic and we + // won't get to see who faulted.) + if r.eip != 0 { + sp := unsafe.Pointer(uintptr(r.esp)) + sp = add(sp, ^uintptr(unsafe.Sizeof(uintptr(0))-1)) // sp-- + *((*uintptr)(sp)) = uintptr(r.eip) + r.esp = uint32(uintptr(sp)) + } + r.eip = uint32(funcPC(sigpanic)) + return _EXCEPTION_CONTINUE_EXECUTION +} + +// lastcontinuehandler is reached, because runtime cannot handle +// current exception. lastcontinuehandler will print crash info and exit. +func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) uint32 { + _g_ := getg() + + if panicking != 0 { // traceback already printed + exit(2) + } + panicking = 1 + + print("Exception ", hex(info.exceptioncode), " ", hex(info.exceptioninformation[0]), " ", hex(info.exceptioninformation[1]), " ", hex(r.eip), "\n") + + print("PC=", hex(r.eip), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + tracebacktrap(uintptr(r.eip), uintptr(r.esp), 0, gp) + tracebackothers(gp) + dumpregs(r) + } + + if docrash { + crash() + } + + exit(2) + return 0 // not reached +} + +func sigenable(sig uint32) { +} + +func sigdisable(sig uint32) { +} + +func dosigprof(r *context, gp *g, mp *m) { + sigprof((*byte)(unsafe.Pointer(uintptr(r.eip))), (*byte)(unsafe.Pointer(uintptr(r.esp))), nil, gp, mp) +} diff --git a/src/runtime/os1_windows_amd64.go b/src/runtime/os1_windows_amd64.go new file mode 100644 index 000000000..0d21b3881 --- /dev/null +++ b/src/runtime/os1_windows_amd64.go @@ -0,0 +1,137 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +var text struct{} + +func dumpregs(r *context) { + print("rax ", hex(r.rax), "\n") + print("rbx ", hex(r.rbx), "\n") + print("rcx ", hex(r.rcx), "\n") + print("rdi ", hex(r.rdi), "\n") + print("rsi ", hex(r.rsi), "\n") + print("rbp ", hex(r.rbp), "\n") + print("rsp ", hex(r.rsp), "\n") + print("r8 ", hex(r.r8), "\n") + print("r9 ", hex(r.r9), "\n") + print("r10 ", hex(r.r10), "\n") + print("r11 ", hex(r.r11), "\n") + print("r12 ", hex(r.r12), "\n") + print("r13 ", hex(r.r13), "\n") + print("r14 ", hex(r.r14), "\n") + print("r15 ", hex(r.r15), "\n") + print("rip ", hex(r.rip), "\n") + print("rflags ", hex(r.eflags), "\n") + print("cs ", hex(r.segcs), "\n") + print("fs ", hex(r.segfs), "\n") + print("gs ", hex(r.seggs), "\n") +} + +func isgoexception(info *exceptionrecord, r *context) bool { + // Only handle exception if executing instructions in Go binary + // (not Windows library code). + if r.rip < uint64(uintptr(unsafe.Pointer(&text))) || uint64(uintptr(unsafe.Pointer(&etext))) < r.rip { + return false + } + + if issigpanic(info.exceptioncode) == 0 { + return false + } + + return true +} + +// Called by sigtramp from Windows VEH handler. +// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION) +// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH). + +func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { + if !isgoexception(info, r) { + return _EXCEPTION_CONTINUE_SEARCH + } + + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = info.exceptioncode + gp.sigcode0 = uintptr(info.exceptioninformation[0]) + gp.sigcode1 = uintptr(info.exceptioninformation[1]) + gp.sigpc = uintptr(r.rip) + + // Only push runtime·sigpanic if r->rip != 0. + // If r->rip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime·sigpanic instead. + // (Otherwise the trace will end at runtime·sigpanic and we + // won't get to see who faulted.) + if r.rip != 0 { + sp := unsafe.Pointer(uintptr(r.rsp)) + sp = add(sp, ^uintptr(unsafe.Sizeof(uintptr(0))-1)) // sp-- + *((*uintptr)(sp)) = uintptr(r.rip) + r.rsp = uint64(uintptr(sp)) + } + r.rip = uint64(funcPC(sigpanic)) + return _EXCEPTION_CONTINUE_EXECUTION +} + +// It seems Windows searches ContinueHandler's list even +// if ExceptionHandler returns EXCEPTION_CONTINUE_EXECUTION. +// firstcontinuehandler will stop that search, +// if exceptionhandler did the same earlier. +func firstcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { + if !isgoexception(info, r) { + return _EXCEPTION_CONTINUE_SEARCH + } + return _EXCEPTION_CONTINUE_EXECUTION +} + +// lastcontinuehandler is reached, because runtime cannot handle +// current exception. lastcontinuehandler will print crash info and exit. +func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) uint32 { + _g_ := getg() + + if panicking != 0 { // traceback already printed + exit(2) + } + panicking = 1 + + print("Exception ", hex(info.exceptioncode), " ", hex(info.exceptioninformation[0]), " ", hex(info.exceptioninformation[1]), " ", hex(r.rip), "\n") + + print("PC=", hex(r.rip), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + tracebacktrap(uintptr(r.rip), uintptr(r.rsp), 0, gp) + tracebackothers(gp) + dumpregs(r) + } + + if docrash { + crash() + } + + exit(2) + return 0 // not reached +} + +func sigenable(sig uint32) { +} + +func sigdisable(sig uint32) { +} + +func dosigprof(r *context, gp *g, mp *m) { + sigprof((*byte)(unsafe.Pointer(uintptr(r.rip))), (*byte)(unsafe.Pointer(uintptr(r.rsp))), nil, gp, mp) +} diff --git a/src/runtime/os2_darwin.go b/src/runtime/os2_darwin.go new file mode 100644 index 000000000..542bd7421 --- /dev/null +++ b/src/runtime/os2_darwin.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _NSIG = 32 + _SI_USER = 0 /* empirically true, but not what headers say */ + _SIG_BLOCK = 1 + _SIG_UNBLOCK = 2 + _SIG_SETMASK = 3 + _SS_DISABLE = 4 +) diff --git a/src/runtime/os2_dragonfly.go b/src/runtime/os2_dragonfly.go new file mode 100644 index 000000000..0a20ed43f --- /dev/null +++ b/src/runtime/os2_dragonfly.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _NSIG = 33 + _SI_USER = 0x10001 + _SS_DISABLE = 4 + _RLIMIT_AS = 10 +) diff --git a/src/runtime/os2_freebsd.go b/src/runtime/os2_freebsd.go new file mode 100644 index 000000000..f67211fdf --- /dev/null +++ b/src/runtime/os2_freebsd.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _SS_DISABLE = 4 + _NSIG = 33 + _SI_USER = 0x10001 + _RLIMIT_AS = 10 +) diff --git a/src/runtime/os2_linux.go b/src/runtime/os2_linux.go new file mode 100644 index 000000000..eaa9f0e83 --- /dev/null +++ b/src/runtime/os2_linux.go @@ -0,0 +1,23 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _SS_DISABLE = 2 + _NSIG = 65 + _SI_USER = 0 + _SIG_SETMASK = 2 + _RLIMIT_AS = 9 +) + +// It's hard to tease out exactly how big a Sigset is, but +// rt_sigprocmask crashes if we get it wrong, so if binaries +// are running, this is right. +type sigset [2]uint32 + +type rlimit struct { + rlim_cur uintptr + rlim_max uintptr +} diff --git a/src/runtime/os2_openbsd.go b/src/runtime/os2_openbsd.go new file mode 100644 index 000000000..1e785ad51 --- /dev/null +++ b/src/runtime/os2_openbsd.go @@ -0,0 +1,14 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _SS_DISABLE = 4 + _SIG_BLOCK = 1 + _SIG_UNBLOCK = 2 + _SIG_SETMASK = 3 + _NSIG = 33 + _SI_USER = 0 +) diff --git a/src/runtime/os2_solaris.go b/src/runtime/os2_solaris.go new file mode 100644 index 000000000..26ca15f62 --- /dev/null +++ b/src/runtime/os2_solaris.go @@ -0,0 +1,13 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _SS_DISABLE = 2 + _SIG_SETMASK = 3 + _NSIG = 73 /* number of signals in sigtable array */ + _SI_USER = 0 + _RLIMIT_AS = 10 +) diff --git a/src/runtime/os2_windows.go b/src/runtime/os2_windows.go new file mode 100644 index 000000000..d5b1f471f --- /dev/null +++ b/src/runtime/os2_windows.go @@ -0,0 +1,25 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Call a Windows function with stdcall conventions, +// and switch to os stack during the call. +func asmstdcall(fn unsafe.Pointer) + +func getlasterror() uint32 +func setlasterror(err uint32) + +// Function to be called by windows CreateThread +// to start new os thread. +func tstart_stdcall(newm *m) uint32 + +func ctrlhandler(_type uint32) uint32 + +// TODO(brainman): should not need those +const ( + _NSIG = 65 +) diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go new file mode 100644 index 000000000..1df74faad --- /dev/null +++ b/src/runtime/os3_solaris.go @@ -0,0 +1,493 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:cgo_export_dynamic runtime.end _end +//go:cgo_export_dynamic runtime.etext _etext +//go:cgo_export_dynamic runtime.edata _edata + +//go:cgo_import_dynamic libc____errno ___errno "libc.so" +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_exit exit "libc.so" +//go:cgo_import_dynamic libc_fstat fstat "libc.so" +//go:cgo_import_dynamic libc_getcontext getcontext "libc.so" +//go:cgo_import_dynamic libc_getrlimit getrlimit "libc.so" +//go:cgo_import_dynamic libc_madvise madvise "libc.so" +//go:cgo_import_dynamic libc_malloc malloc "libc.so" +//go:cgo_import_dynamic libc_mmap mmap "libc.so" +//go:cgo_import_dynamic libc_munmap munmap "libc.so" +//go:cgo_import_dynamic libc_open open "libc.so" +//go:cgo_import_dynamic libc_pthread_attr_destroy pthread_attr_destroy "libc.so" +//go:cgo_import_dynamic libc_pthread_attr_getstack pthread_attr_getstack "libc.so" +//go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "libc.so" +//go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "libc.so" +//go:cgo_import_dynamic libc_pthread_attr_setstack pthread_attr_setstack "libc.so" +//go:cgo_import_dynamic libc_pthread_create pthread_create "libc.so" +//go:cgo_import_dynamic libc_raise raise "libc.so" +//go:cgo_import_dynamic libc_read read "libc.so" +//go:cgo_import_dynamic libc_select select "libc.so" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" +//go:cgo_import_dynamic libc_sem_init sem_init "libc.so" +//go:cgo_import_dynamic libc_sem_post sem_post "libc.so" +//go:cgo_import_dynamic libc_sem_reltimedwait_np sem_reltimedwait_np "libc.so" +//go:cgo_import_dynamic libc_sem_wait sem_wait "libc.so" +//go:cgo_import_dynamic libc_setitimer setitimer "libc.so" +//go:cgo_import_dynamic libc_sigaction sigaction "libc.so" +//go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" +//go:cgo_import_dynamic libc_sigprocmask sigprocmask "libc.so" +//go:cgo_import_dynamic libc_sysconf sysconf "libc.so" +//go:cgo_import_dynamic libc_usleep usleep "libc.so" +//go:cgo_import_dynamic libc_write write "libc.so" + +//go:linkname libc____errno libc____errno +//go:linkname libc_clock_gettime libc_clock_gettime +//go:linkname libc_close libc_close +//go:linkname libc_exit libc_exit +//go:linkname libc_fstat libc_fstat +//go:linkname libc_getcontext libc_getcontext +//go:linkname libc_getrlimit libc_getrlimit +//go:linkname libc_madvise libc_madvise +//go:linkname libc_malloc libc_malloc +//go:linkname libc_mmap libc_mmap +//go:linkname libc_munmap libc_munmap +//go:linkname libc_open libc_open +//go:linkname libc_pthread_attr_destroy libc_pthread_attr_destroy +//go:linkname libc_pthread_attr_getstack libc_pthread_attr_getstack +//go:linkname libc_pthread_attr_init libc_pthread_attr_init +//go:linkname libc_pthread_attr_setdetachstate libc_pthread_attr_setdetachstate +//go:linkname libc_pthread_attr_setstack libc_pthread_attr_setstack +//go:linkname libc_pthread_create libc_pthread_create +//go:linkname libc_raise libc_raise +//go:linkname libc_read libc_read +//go:linkname libc_select libc_select +//go:linkname libc_sched_yield libc_sched_yield +//go:linkname libc_sem_init libc_sem_init +//go:linkname libc_sem_post libc_sem_post +//go:linkname libc_sem_reltimedwait_np libc_sem_reltimedwait_np +//go:linkname libc_sem_wait libc_sem_wait +//go:linkname libc_setitimer libc_setitimer +//go:linkname libc_sigaction libc_sigaction +//go:linkname libc_sigaltstack libc_sigaltstack +//go:linkname libc_sigprocmask libc_sigprocmask +//go:linkname libc_sysconf libc_sysconf +//go:linkname libc_usleep libc_usleep +//go:linkname libc_write libc_write + +var ( + libc____errno, + libc_clock_gettime, + libc_close, + libc_exit, + libc_fstat, + libc_getcontext, + libc_getrlimit, + libc_madvise, + libc_malloc, + libc_mmap, + libc_munmap, + libc_open, + libc_pthread_attr_destroy, + libc_pthread_attr_getstack, + libc_pthread_attr_init, + libc_pthread_attr_setdetachstate, + libc_pthread_attr_setstack, + libc_pthread_create, + libc_raise, + libc_read, + libc_sched_yield, + libc_select, + libc_sem_init, + libc_sem_post, + libc_sem_reltimedwait_np, + libc_sem_wait, + libc_setitimer, + libc_sigaction, + libc_sigaltstack, + libc_sigprocmask, + libc_sysconf, + libc_usleep, + libc_write libcFunc +) + +var sigset_none = sigset{} +var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} + +func getncpu() int32 { + n := int32(sysconf(__SC_NPROCESSORS_ONLN)) + if n < 1 { + return 1 + } + return n +} + +func osinit() { + ncpu = getncpu() +} + +func tstart_sysvicall() + +func newosproc(mp *m, _ unsafe.Pointer) { + var ( + attr pthreadattr + oset sigset + tid pthread + ret int32 + size uint64 + ) + + if pthread_attr_init(&attr) != 0 { + gothrow("pthread_attr_init") + } + if pthread_attr_setstack(&attr, 0, 0x200000) != 0 { + gothrow("pthread_attr_setstack") + } + if pthread_attr_getstack(&attr, unsafe.Pointer(&mp.g0.stack.hi), &size) != 0 { + gothrow("pthread_attr_getstack") + } + mp.g0.stack.lo = mp.g0.stack.hi - uintptr(size) + if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { + gothrow("pthread_attr_setdetachstate") + } + + // Disable signals during create, so that the new thread starts + // with signals disabled. It will enable them in minit. + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + ret = pthread_create(&tid, &attr, funcPC(tstart_sysvicall), unsafe.Pointer(mp)) + sigprocmask(_SIG_SETMASK, &oset, nil) + if ret != 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n") + gothrow("newosproc") + } +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) + mp.gsignal.m = mp +} + +func miniterrno() + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + _g_ := getg() + asmcgocall(unsafe.Pointer(funcPC(miniterrno)), unsafe.Pointer(libc____errno)) + // Initialize signal handling + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + /* + TODO: Convert to Go when something actually uses the result. + Rlimit rl; + extern byte runtime·text[], runtime·end[]; + uintptr used; + + if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) + return 0; + if(rl.rlim_cur >= 0x7fffffff) + return 0; + + // Estimate our VM footprint excluding the heap. + // Not an exact science: use size of binary plus + // some room for thread stacks. + used = runtime·end - runtime·text + (64<<20); + if(used >= rl.rlim_cur) + return 0; + + // If there's not at least 16 MB left, we're probably + // not going to be able to do much. Treat as no limit. + rl.rlim_cur -= used; + if(rl.rlim_cur < (16<<20)) + return 0; + + return rl.rlim_cur - used; + */ + + return 0 +} + +func sigtramp() + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = sigset_all + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + *((*uintptr)(unsafe.Pointer(&sa._funcptr))) = fn + sigaction(i, &sa, nil) +} + +func getsig(i int32) uintptr { + var sa sigactiont + sigaction(i, nil, &sa) + if *((*uintptr)(unsafe.Pointer(&sa._funcptr))) == funcPC(sigtramp) { + return funcPC(sighandler) + } + return *((*uintptr)(unsafe.Pointer(&sa._funcptr))) +} + +func signalstack(p *byte, n int32) { + var st sigaltstackt + st.ss_sp = (*byte)(unsafe.Pointer(p)) + st.ss_size = uint64(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} + +//go:nosplit +func semacreate() uintptr { + var sem *semt + _g_ := getg() + + // Call libc's malloc rather than malloc. This will + // allocate space on the C heap. We can't call malloc + // here because it could cause a deadlock. + _g_.m.libcall.fn = uintptr(libc_malloc) + _g_.m.libcall.n = 1 + memclr(unsafe.Pointer(&_g_.m.scratch), uintptr(len(_g_.m.scratch.v))) + _g_.m.scratch.v[0] = unsafe.Sizeof(*sem) + _g_.m.libcall.args = uintptr(unsafe.Pointer(&_g_.m.scratch)) + asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_g_.m.libcall)) + sem = (*semt)(unsafe.Pointer(_g_.m.libcall.r1)) + if sem_init(sem, 0, 0) != 0 { + gothrow("sem_init") + } + return uintptr(unsafe.Pointer(sem)) +} + +//go:nosplit +func semasleep(ns int64) int32 { + _m_ := getg().m + if ns >= 0 { + _m_.ts.tv_sec = ns / 1000000000 + _m_.ts.tv_nsec = ns % 1000000000 + + _m_.libcall.fn = uintptr(unsafe.Pointer(libc_sem_reltimedwait_np)) + _m_.libcall.n = 2 + memclr(unsafe.Pointer(&_m_.scratch), uintptr(len(_m_.scratch.v))) + _m_.scratch.v[0] = _m_.waitsema + _m_.scratch.v[1] = uintptr(unsafe.Pointer(&_m_.ts)) + _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) + asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) + if *_m_.perrno != 0 { + if *_m_.perrno == _ETIMEDOUT || *_m_.perrno == _EAGAIN || *_m_.perrno == _EINTR { + return -1 + } + gothrow("sem_reltimedwait_np") + } + return 0 + } + for { + _m_.libcall.fn = uintptr(unsafe.Pointer(libc_sem_wait)) + _m_.libcall.n = 1 + memclr(unsafe.Pointer(&_m_.scratch), uintptr(len(_m_.scratch.v))) + _m_.scratch.v[0] = _m_.waitsema + _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) + asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) + if _m_.libcall.r1 == 0 { + break + } + if *_m_.perrno == _EINTR { + continue + } + gothrow("sem_wait") + } + return 0 +} + +//go:nosplit +func semawakeup(mp *m) { + if sem_post((*semt)(unsafe.Pointer(mp.waitsema))) != 0 { + gothrow("sem_post") + } +} + +//go:nosplit +func close(fd int32) int32 { + return int32(sysvicall1(libc_close, uintptr(fd))) +} + +//go:nosplit +func exit(r int32) { + sysvicall1(libc_exit, uintptr(r)) +} + +//go:nosplit +func getcontext(context *ucontext) /* int32 */ { + sysvicall1(libc_getcontext, uintptr(unsafe.Pointer(context))) +} + +//go:nosplit +func madvise(addr unsafe.Pointer, n uintptr, flags int32) { + sysvicall3(libc_madvise, uintptr(addr), uintptr(n), uintptr(flags)) +} + +//go:nosplit +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer { + return unsafe.Pointer(sysvicall6(libc_mmap, uintptr(addr), uintptr(n), uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off))) +} + +//go:nosplit +func munmap(addr unsafe.Pointer, n uintptr) { + sysvicall2(libc_munmap, uintptr(addr), uintptr(n)) +} + +func nanotime1() + +//go:nosplit +func nanotime() int64 { + return int64(sysvicall0(libcFunc(funcPC(nanotime1)))) +} + +//go:nosplit +func open(path *byte, mode, perm int32) int32 { + return int32(sysvicall3(libc_open, uintptr(unsafe.Pointer(path)), uintptr(mode), uintptr(perm))) +} + +func pthread_attr_destroy(attr *pthreadattr) int32 { + return int32(sysvicall1(libc_pthread_attr_destroy, uintptr(unsafe.Pointer(attr)))) +} + +func pthread_attr_getstack(attr *pthreadattr, addr unsafe.Pointer, size *uint64) int32 { + return int32(sysvicall3(libc_pthread_attr_getstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(unsafe.Pointer(size)))) +} + +func pthread_attr_init(attr *pthreadattr) int32 { + return int32(sysvicall1(libc_pthread_attr_init, uintptr(unsafe.Pointer(attr)))) +} + +func pthread_attr_setdetachstate(attr *pthreadattr, state int32) int32 { + return int32(sysvicall2(libc_pthread_attr_setdetachstate, uintptr(unsafe.Pointer(attr)), uintptr(state))) +} + +func pthread_attr_setstack(attr *pthreadattr, addr uintptr, size uint64) int32 { + return int32(sysvicall3(libc_pthread_attr_setstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(size))) +} + +func pthread_create(thread *pthread, attr *pthreadattr, fn uintptr, arg unsafe.Pointer) int32 { + return int32(sysvicall4(libc_pthread_create, uintptr(unsafe.Pointer(thread)), uintptr(unsafe.Pointer(attr)), uintptr(fn), uintptr(arg))) +} + +func raise(sig int32) /* int32 */ { + sysvicall1(libc_raise, uintptr(sig)) +} + +//go:nosplit +func read(fd int32, buf unsafe.Pointer, nbyte int32) int32 { + return int32(sysvicall3(libc_read, uintptr(fd), uintptr(buf), uintptr(nbyte))) +} + +//go:nosplit +func sem_init(sem *semt, pshared int32, value uint32) int32 { + return int32(sysvicall3(libc_sem_init, uintptr(unsafe.Pointer(sem)), uintptr(pshared), uintptr(value))) +} + +//go:nosplit +func sem_post(sem *semt) int32 { + return int32(sysvicall1(libc_sem_post, uintptr(unsafe.Pointer(sem)))) +} + +//go:nosplit +func sem_reltimedwait_np(sem *semt, timeout *timespec) int32 { + return int32(sysvicall2(libc_sem_reltimedwait_np, uintptr(unsafe.Pointer(sem)), uintptr(unsafe.Pointer(timeout)))) +} + +//go:nosplit +func sem_wait(sem *semt) int32 { + return int32(sysvicall1(libc_sem_wait, uintptr(unsafe.Pointer(sem)))) +} + +func setitimer(which int32, value *itimerval, ovalue *itimerval) /* int32 */ { + sysvicall3(libc_setitimer, uintptr(which), uintptr(unsafe.Pointer(value)), uintptr(unsafe.Pointer(ovalue))) +} + +func sigaction(sig int32, act *sigactiont, oact *sigactiont) /* int32 */ { + sysvicall3(libc_sigaction, uintptr(sig), uintptr(unsafe.Pointer(act)), uintptr(unsafe.Pointer(oact))) +} + +func sigaltstack(ss *sigaltstackt, oss *sigaltstackt) /* int32 */ { + sysvicall2(libc_sigaltstack, uintptr(unsafe.Pointer(ss)), uintptr(unsafe.Pointer(oss))) +} + +func sigprocmask(how int32, set *sigset, oset *sigset) /* int32 */ { + sysvicall3(libc_sigprocmask, uintptr(how), uintptr(unsafe.Pointer(set)), uintptr(unsafe.Pointer(oset))) +} + +func sysconf(name int32) int64 { + return int64(sysvicall1(libc_sysconf, uintptr(name))) +} + +func usleep1(uint32) + +//go:nosplit +func usleep(µs uint32) { + usleep1(µs) +} + +//go:nosplit +func write(fd uintptr, buf unsafe.Pointer, nbyte int32) int32 { + return int32(sysvicall3(libc_write, uintptr(fd), uintptr(buf), uintptr(nbyte))) +} + +func osyield1() + +//go:nosplit +func osyield() { + _g_ := getg() + + // Check the validity of m because we might be called in cgo callback + // path early enough where there isn't a m available yet. + if _g_ != nil && _g_.m != nil { + sysvicall0(libc_sched_yield) + return + } + osyield1() +} diff --git a/src/runtime/os_darwin.c b/src/runtime/os_darwin.c deleted file mode 100644 index bbd29282b..000000000 --- a/src/runtime/os_darwin.c +++ /dev/null @@ -1,567 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; - -static Sigset sigset_none; -static Sigset sigset_all = ~(Sigset)0; - -static void -unimplemented(int8 *name) -{ - runtime·prints(name); - runtime·prints(" not implemented\n"); - *(int32*)1231 = 1231; -} - -#pragma textflag NOSPLIT -void -runtime·semawakeup(M *mp) -{ - runtime·mach_semrelease(mp->waitsema); -} - -static void -semacreate(void) -{ - g->m->scalararg[0] = runtime·mach_semcreate(); -} - -#pragma textflag NOSPLIT -uintptr -runtime·semacreate(void) -{ - uintptr x; - void (*fn)(void); - - fn = semacreate; - runtime·onM(&fn); - x = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - return x; -} - -// BSD interface for threading. -void -runtime·osinit(void) -{ - // bsdthread_register delayed until end of goenvs so that we - // can look at the environment first. - - // Use sysctl to fetch hw.ncpu. - uint32 mib[2]; - uint32 out; - int32 ret; - uintptr nout; - - mib[0] = 6; - mib[1] = 3; - nout = sizeof out; - out = 0; - ret = runtime·sysctl(mib, 2, (byte*)&out, &nout, nil, 0); - if(ret >= 0) - runtime·ncpu = out; -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); - - // Register our thread-creation callback (see sys_darwin_{amd64,386}.s) - // but only if we're not using cgo. If we are using cgo we need - // to let the C pthread library install its own thread-creation callback. - if(!runtime·iscgo) { - if(runtime·bsdthread_register() != 0) { - if(runtime·getenv("DYLD_INSERT_LIBRARIES")) - runtime·throw("runtime: bsdthread_register error (unset DYLD_INSERT_LIBRARIES)"); - runtime·throw("runtime: bsdthread_register error"); - } - } - -} - -void -runtime·newosproc(M *mp, void *stk) -{ - int32 errno; - Sigset oset; - - mp->tls[0] = mp->id; // so 386 asm can find it - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, mp->id, (int32)mp->tls[0], &mp); - } - - runtime·sigprocmask(SIG_SETMASK, &sigset_all, &oset); - errno = runtime·bsdthread_create(stk, mp, mp->g0, runtime·mstart); - runtime·sigprocmask(SIG_SETMASK, &oset, nil); - - if(errno < 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -errno); - runtime·throw("runtime.newosproc"); - } -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling. - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -// Mach IPC, to get at semaphores -// Definitions are in /usr/include/mach on a Mac. - -static void -macherror(int32 r, int8 *fn) -{ - runtime·prints("mach error "); - runtime·prints(fn); - runtime·prints(": "); - runtime·printint(r); - runtime·prints("\n"); - runtime·throw("mach error"); -} - -enum -{ - DebugMach = 0 -}; - -static MachNDR zerondr; - -#define MACH_MSGH_BITS(a, b) ((a) | ((b)<<8)) - -static int32 -mach_msg(MachHeader *h, - int32 op, - uint32 send_size, - uint32 rcv_size, - uint32 rcv_name, - uint32 timeout, - uint32 notify) -{ - // TODO: Loop on interrupt. - return runtime·mach_msg_trap(h, op, send_size, rcv_size, rcv_name, timeout, notify); -} - -// Mach RPC (MIG) - -enum -{ - MinMachMsg = 48, - Reply = 100, -}; - -#pragma pack on -typedef struct CodeMsg CodeMsg; -struct CodeMsg -{ - MachHeader h; - MachNDR NDR; - int32 code; -}; -#pragma pack off - -static int32 -machcall(MachHeader *h, int32 maxsize, int32 rxsize) -{ - uint32 *p; - int32 i, ret, id; - uint32 port; - CodeMsg *c; - - if((port = g->m->machport) == 0){ - port = runtime·mach_reply_port(); - g->m->machport = port; - } - - h->msgh_bits |= MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND_ONCE); - h->msgh_local_port = port; - h->msgh_reserved = 0; - id = h->msgh_id; - - if(DebugMach){ - p = (uint32*)h; - runtime·prints("send:\t"); - for(i=0; i<h->msgh_size/sizeof(p[0]); i++){ - runtime·prints(" "); - runtime·printpointer((void*)p[i]); - if(i%8 == 7) - runtime·prints("\n\t"); - } - if(i%8) - runtime·prints("\n"); - } - - ret = mach_msg(h, MACH_SEND_MSG|MACH_RCV_MSG, - h->msgh_size, maxsize, port, 0, 0); - if(ret != 0){ - if(DebugMach){ - runtime·prints("mach_msg error "); - runtime·printint(ret); - runtime·prints("\n"); - } - return ret; - } - - if(DebugMach){ - p = (uint32*)h; - runtime·prints("recv:\t"); - for(i=0; i<h->msgh_size/sizeof(p[0]); i++){ - runtime·prints(" "); - runtime·printpointer((void*)p[i]); - if(i%8 == 7) - runtime·prints("\n\t"); - } - if(i%8) - runtime·prints("\n"); - } - - if(h->msgh_id != id+Reply){ - if(DebugMach){ - runtime·prints("mach_msg reply id mismatch "); - runtime·printint(h->msgh_id); - runtime·prints(" != "); - runtime·printint(id+Reply); - runtime·prints("\n"); - } - return -303; // MIG_REPLY_MISMATCH - } - - // Look for a response giving the return value. - // Any call can send this back with an error, - // and some calls only have return values so they - // send it back on success too. I don't quite see how - // you know it's one of these and not the full response - // format, so just look if the message is right. - c = (CodeMsg*)h; - if(h->msgh_size == sizeof(CodeMsg) - && !(h->msgh_bits & MACH_MSGH_BITS_COMPLEX)){ - if(DebugMach){ - runtime·prints("mig result "); - runtime·printint(c->code); - runtime·prints("\n"); - } - return c->code; - } - - if(h->msgh_size != rxsize){ - if(DebugMach){ - runtime·prints("mach_msg reply size mismatch "); - runtime·printint(h->msgh_size); - runtime·prints(" != "); - runtime·printint(rxsize); - runtime·prints("\n"); - } - return -307; // MIG_ARRAY_TOO_LARGE - } - - return 0; -} - - -// Semaphores! - -enum -{ - Tmach_semcreate = 3418, - Rmach_semcreate = Tmach_semcreate + Reply, - - Tmach_semdestroy = 3419, - Rmach_semdestroy = Tmach_semdestroy + Reply, - - // Mach calls that get interrupted by Unix signals - // return this error code. We retry them. - KERN_ABORTED = 14, - KERN_OPERATION_TIMED_OUT = 49, -}; - -typedef struct Tmach_semcreateMsg Tmach_semcreateMsg; -typedef struct Rmach_semcreateMsg Rmach_semcreateMsg; -typedef struct Tmach_semdestroyMsg Tmach_semdestroyMsg; -// Rmach_semdestroyMsg = CodeMsg - -#pragma pack on -struct Tmach_semcreateMsg -{ - MachHeader h; - MachNDR ndr; - int32 policy; - int32 value; -}; - -struct Rmach_semcreateMsg -{ - MachHeader h; - MachBody body; - MachPort semaphore; -}; - -struct Tmach_semdestroyMsg -{ - MachHeader h; - MachBody body; - MachPort semaphore; -}; -#pragma pack off - -uint32 -runtime·mach_semcreate(void) -{ - union { - Tmach_semcreateMsg tx; - Rmach_semcreateMsg rx; - uint8 pad[MinMachMsg]; - } m; - int32 r; - - m.tx.h.msgh_bits = 0; - m.tx.h.msgh_size = sizeof(m.tx); - m.tx.h.msgh_remote_port = runtime·mach_task_self(); - m.tx.h.msgh_id = Tmach_semcreate; - m.tx.ndr = zerondr; - - m.tx.policy = 0; // 0 = SYNC_POLICY_FIFO - m.tx.value = 0; - - while((r = machcall(&m.tx.h, sizeof m, sizeof(m.rx))) != 0){ - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_create"); - } - if(m.rx.body.msgh_descriptor_count != 1) - unimplemented("mach_semcreate desc count"); - return m.rx.semaphore.name; -} - -void -runtime·mach_semdestroy(uint32 sem) -{ - union { - Tmach_semdestroyMsg tx; - uint8 pad[MinMachMsg]; - } m; - int32 r; - - m.tx.h.msgh_bits = MACH_MSGH_BITS_COMPLEX; - m.tx.h.msgh_size = sizeof(m.tx); - m.tx.h.msgh_remote_port = runtime·mach_task_self(); - m.tx.h.msgh_id = Tmach_semdestroy; - m.tx.body.msgh_descriptor_count = 1; - m.tx.semaphore.name = sem; - m.tx.semaphore.disposition = MACH_MSG_TYPE_MOVE_SEND; - m.tx.semaphore.type = 0; - - while((r = machcall(&m.tx.h, sizeof m, 0)) != 0){ - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_destroy"); - } -} - -// The other calls have simple system call traps in sys_darwin_{amd64,386}.s -int32 runtime·mach_semaphore_wait(uint32 sema); -int32 runtime·mach_semaphore_timedwait(uint32 sema, uint32 sec, uint32 nsec); -int32 runtime·mach_semaphore_signal(uint32 sema); -int32 runtime·mach_semaphore_signal_all(uint32 sema); - -static void -semasleep(void) -{ - int32 r, secs, nsecs; - int64 ns; - - ns = (int64)(uint32)g->m->scalararg[0] | (int64)(uint32)g->m->scalararg[1]<<32; - g->m->scalararg[0] = 0; - g->m->scalararg[1] = 0; - - if(ns >= 0) { - secs = runtime·timediv(ns, 1000000000, &nsecs); - r = runtime·mach_semaphore_timedwait(g->m->waitsema, secs, nsecs); - if(r == KERN_ABORTED || r == KERN_OPERATION_TIMED_OUT) { - g->m->scalararg[0] = -1; - return; - } - if(r != 0) - macherror(r, "semaphore_wait"); - g->m->scalararg[0] = 0; - return; - } - while((r = runtime·mach_semaphore_wait(g->m->waitsema)) != 0) { - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_wait"); - } - g->m->scalararg[0] = 0; - return; -} - -#pragma textflag NOSPLIT -int32 -runtime·semasleep(int64 ns) -{ - int32 r; - void (*fn)(void); - - g->m->scalararg[0] = (uint32)ns; - g->m->scalararg[1] = (uint32)(ns>>32); - fn = semasleep; - runtime·onM(&fn); - r = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - return r; -} - -static int32 mach_semrelease_errno; - -static void -mach_semrelease_fail(void) -{ - macherror(mach_semrelease_errno, "semaphore_signal"); -} - -#pragma textflag NOSPLIT -void -runtime·mach_semrelease(uint32 sem) -{ - int32 r; - void (*fn)(void); - - while((r = runtime·mach_semaphore_signal(sem)) != 0) { - if(r == KERN_ABORTED) // interrupted - continue; - - // mach_semrelease must be completely nosplit, - // because it is called from Go code. - // If we're going to die, start that process on the m stack - // to avoid a Go stack split. - // Only do that if we're actually running on the g stack. - // We might be on the gsignal stack, and if so, onM will abort. - // We use the global variable instead of scalararg because - // we might be on the gsignal stack, having interrupted a - // normal call to onM. It doesn't quite matter, since the - // program is about to die, but better to be clean. - mach_semrelease_errno = r; - fn = mach_semrelease_fail; - if(g == g->m->curg) - runtime·onM(&fn); - else - fn(); - } -} - -#pragma textflag NOSPLIT -void -runtime·osyield(void) -{ - runtime·usleep(1); -} - -uintptr -runtime·memlimit(void) -{ - // NOTE(rsc): Could use getrlimit here, - // like on FreeBSD or Linux, but Darwin doesn't enforce - // ulimit -v, so it's unclear why we'd try to stay within - // the limit. - return 0; -} - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~(uintptr)0; - sa.sa_tramp = (void*)runtime·sigtramp; // runtime·sigtramp's job is to call into real handler - *(uintptr*)sa.__sigaction_u = (uintptr)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - return *(void**)sa.__sigaction_u; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 4327ced91..d8296e056 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -6,19 +6,31 @@ package runtime import "unsafe" -func bsdthread_create(stk, mm, gg, fn unsafe.Pointer) int32 +func bsdthread_create(stk unsafe.Pointer, mm *m, gg *g, fn uintptr) int32 func bsdthread_register() int32 + +//go:noescape func mach_msg_trap(h unsafe.Pointer, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32 + func mach_reply_port() uint32 func mach_task_self() uint32 func mach_thread_self() uint32 + +//go:noescape func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 -func sigprocmask(sig int32, new, old unsafe.Pointer) -func sigaction(mode uint32, new, old unsafe.Pointer) -func sigaltstack(new, old unsafe.Pointer) + +//go:noescape +func sigprocmask(sig uint32, new, old *uint32) + +//go:noescape +func sigaction(mode uint32, new, old *sigactiont) + +//go:noescape +func sigaltstack(new, old *stackt) + func sigtramp() -func setitimer(mode int32, new, old unsafe.Pointer) -func mach_semaphore_wait(sema uint32) int32 -func mach_semaphore_timedwait(sema, sec, nsec uint32) int32 -func mach_semaphore_signal(sema uint32) int32 -func mach_semaphore_signal_all(sema uint32) int32 + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +func raise(int32) diff --git a/src/runtime/os_darwin.h b/src/runtime/os_darwin.h deleted file mode 100644 index e8bb45daf..000000000 --- a/src/runtime/os_darwin.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -typedef byte* kevent_udata; - -int32 runtime·bsdthread_create(void*, M*, G*, void(*)(void)); -int32 runtime·bsdthread_register(void); -int32 runtime·mach_msg_trap(MachHeader*, int32, uint32, uint32, uint32, uint32, uint32); -uint32 runtime·mach_reply_port(void); -int32 runtime·mach_semacquire(uint32, int64); -uint32 runtime·mach_semcreate(void); -void runtime·mach_semdestroy(uint32); -void runtime·mach_semrelease(uint32); -void runtime·mach_semreset(uint32); -uint32 runtime·mach_task_self(void); -uint32 runtime·mach_task_self(void); -uint32 runtime·mach_thread_self(void); -uint32 runtime·mach_thread_self(void); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - -typedef uint32 Sigset; -void runtime·sigprocmask(int32, Sigset*, Sigset*); -void runtime·unblocksignals(void); - -struct SigactionT; -void runtime·sigaction(uintptr, struct SigactionT*, struct SigactionT*); - -struct StackT; -void runtime·sigaltstack(struct StackT*, struct StackT*); -void runtime·sigtramp(void); -void runtime·sigpanic(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); - - -enum { - NSIG = 32, - SI_USER = 0, /* empirically true, but not what headers say */ - SIG_BLOCK = 1, - SIG_UNBLOCK = 2, - SIG_SETMASK = 3, - SS_DISABLE = 4, -}; diff --git a/src/runtime/os_dragonfly.c b/src/runtime/os_dragonfly.c deleted file mode 100644 index e372205ec..000000000 --- a/src/runtime/os_dragonfly.c +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; -extern int32 runtime·sys_umtx_sleep(uint32*, int32, int32); -extern int32 runtime·sys_umtx_wakeup(uint32*, int32); - -// From DragonFly's <sys/sysctl.h> -#define CTL_HW 6 -#define HW_NCPU 3 - -static Sigset sigset_none; -static Sigset sigset_all = { ~(uint32)0, ~(uint32)0, ~(uint32)0, ~(uint32)0, }; - -static int32 -getncpu(void) -{ - uint32 mib[2]; - uint32 out; - int32 ret; - uintptr nout; - - // Fetch hw.ncpu via sysctl. - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - nout = sizeof out; - out = 0; - ret = runtime·sysctl(mib, 2, (byte*)&out, &nout, nil, 0); - if(ret >= 0) - return out; - else - return 1; -} - -static void futexsleep(void); - -#pragma textflag NOSPLIT -void -runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) -{ - void (*fn)(void); - - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = val; - g->m->ptrarg[1] = &ns; - - fn = futexsleep; - runtime·onM(&fn); -} - -static void -futexsleep(void) -{ - uint32 *addr; - uint32 val; - int64 ns; - int32 timeout = 0; - int32 ret; - - addr = g->m->ptrarg[0]; - val = g->m->scalararg[0]; - ns = *(int64*)g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->scalararg[0] = 0; - g->m->ptrarg[1] = nil; - - if(ns >= 0) { - // The timeout is specified in microseconds - ensure that we - // do not end up dividing to zero, which would put us to sleep - // indefinitely... - timeout = runtime·timediv(ns, 1000, nil); - if(timeout == 0) - timeout = 1; - } - - // sys_umtx_sleep will return EWOULDBLOCK (EAGAIN) when the timeout - // expires or EBUSY if the mutex value does not match. - ret = runtime·sys_umtx_sleep(addr, val, timeout); - if(ret >= 0 || ret == -EINTR || ret == -EAGAIN || ret == -EBUSY) - return; - - runtime·prints("umtx_wait addr="); - runtime·printpointer(addr); - runtime·prints(" val="); - runtime·printint(val); - runtime·prints(" ret="); - runtime·printint(ret); - runtime·prints("\n"); - *(int32*)0x1005 = 0x1005; -} - -static void badfutexwakeup(void); - -#pragma textflag NOSPLIT -void -runtime·futexwakeup(uint32 *addr, uint32 cnt) -{ - int32 ret; - void (*fn)(void); - - ret = runtime·sys_umtx_wakeup(addr, cnt); - if(ret >= 0) - return; - - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = ret; - fn = badfutexwakeup; - if(g == g->m->gsignal) - fn(); - else - runtime·onM(&fn); - *(int32*)0x1006 = 0x1006; -} - -static void -badfutexwakeup(void) -{ - void *addr; - int32 ret; - - addr = g->m->ptrarg[0]; - ret = g->m->scalararg[0]; - runtime·printf("umtx_wake addr=%p ret=%d\n", addr, ret); -} - -void runtime·lwp_start(void*); - -void -runtime·newosproc(M *mp, void *stk) -{ - Lwpparams params; - Sigset oset; - - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, mp->id, (int32)mp->tls[0], &mp); - } - - runtime·sigprocmask(&sigset_all, &oset); - runtime·memclr((byte*)¶ms, sizeof params); - - params.func = runtime·lwp_start; - params.arg = (byte*)mp; - params.stack = (byte*)stk; - params.tid1 = (int32*)&mp->procid; - params.tid2 = nil; - - mp->tls[0] = mp->id; // so 386 asm can find it - - runtime·lwp_create(¶ms); - runtime·sigprocmask(&oset, nil); -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getncpu(); -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·sigprocmask(&sigset_none, nil); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - Rlimit rl; - extern byte runtime·text[], runtime·end[]; - uintptr used; - - if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) - return 0; - if(rl.rlim_cur >= 0x7fffffff) - return 0; - - // Estimate our VM footprint excluding the heap. - // Not an exact science: use size of binary plus - // some room for thread stacks. - used = runtime·end - runtime·text + (64<<20); - if(used >= rl.rlim_cur) - return 0; - - // If there's not at least 16 MB left, we're probably - // not going to be able to do much. Treat as no limit. - rl.rlim_cur -= used; - if(rl.rlim_cur < (16<<20)) - return 0; - - return rl.rlim_cur - used; -} - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - int32 sa_flags; /* see signal options below */ - Sigset sa_mask; /* signal mask to apply */ -} SigactionT; - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__bits[0] = ~(uint32)0; - sa.sa_mask.__bits[1] = ~(uint32)0; - sa.sa_mask.__bits[2] = ~(uint32)0; - sa.sa_mask.__bits[3] = ~(uint32)0; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if((void*)sa.__sigaction_u.__sa_sigaction == runtime·sigtramp) - return runtime·sighandler; - return (void*)sa.__sigaction_u.__sa_sigaction; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(&sigset_none, nil); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_dragonfly.go b/src/runtime/os_dragonfly.go index cdaa06986..0e00f874f 100644 --- a/src/runtime/os_dragonfly.go +++ b/src/runtime/os_dragonfly.go @@ -6,15 +6,35 @@ package runtime import "unsafe" -func lwp_create(param unsafe.Pointer) int32 -func sigaltstack(new, old unsafe.Pointer) -func sigaction(sig int32, new, old unsafe.Pointer) -func sigprocmask(new, old unsafe.Pointer) -func setitimer(mode int32, new, old unsafe.Pointer) +//go:noescape +func lwp_create(param *lwpparams) int32 + +//go:noescape +func sigaltstack(new, old *sigaltstackt) + +//go:noescape +func sigaction(sig int32, new, old *sigactiont) + +//go:noescape +func sigprocmask(new, old *sigset) + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +//go:noescape func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 + +//go:noescape func getrlimit(kind int32, limit unsafe.Pointer) int32 + func raise(sig int32) -func sys_umtx_sleep(addr unsafe.Pointer, val, timeout int32) int32 -func sys_umtx_wakeup(addr unsafe.Pointer, val int32) int32 + +//go:noescape +func sys_umtx_sleep(addr *uint32, val, timeout int32) int32 + +//go:noescape +func sys_umtx_wakeup(addr *uint32, val int32) int32 + +func osyield() const stackSystem = 0 diff --git a/src/runtime/os_dragonfly.h b/src/runtime/os_dragonfly.h deleted file mode 100644 index 389736a32..000000000 --- a/src/runtime/os_dragonfly.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - - -typedef byte* kevent_udata; - -int32 runtime·lwp_create(Lwpparams*); -void runtime·sigpanic(void); -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -struct sigaction; -void runtime·sigaction(int32, struct sigaction*, struct sigaction*); -void runtime·sigprocmask(Sigset *, Sigset *); -void runtime·unblocksignals(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - -enum { - NSIG = 33, - SI_USER = 0x10001, - SS_DISABLE = 4, - RLIMIT_AS = 10, -}; - -typedef struct Rlimit Rlimit; -struct Rlimit { - int64 rlim_cur; - int64 rlim_max; -}; -int32 runtime·getrlimit(int32, Rlimit*); diff --git a/src/runtime/os_freebsd.c b/src/runtime/os_freebsd.c deleted file mode 100644 index a513cb604..000000000 --- a/src/runtime/os_freebsd.c +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; -extern int32 runtime·sys_umtx_op(uint32*, int32, uint32, void*, void*); - -// From FreeBSD's <sys/sysctl.h> -#define CTL_HW 6 -#define HW_NCPU 3 - -static Sigset sigset_none; -static Sigset sigset_all = { ~(uint32)0, ~(uint32)0, ~(uint32)0, ~(uint32)0, }; - -static int32 -getncpu(void) -{ - uint32 mib[2]; - uint32 out; - int32 ret; - uintptr nout; - - // Fetch hw.ncpu via sysctl. - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - nout = sizeof out; - out = 0; - ret = runtime·sysctl(mib, 2, (byte*)&out, &nout, nil, 0); - if(ret >= 0) - return out; - else - return 1; -} - -// FreeBSD's umtx_op syscall is effectively the same as Linux's futex, and -// thus the code is largely similar. See linux/thread.c and lock_futex.c for comments. - -static void futexsleep(void); - -#pragma textflag NOSPLIT -void -runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) -{ - void (*fn)(void); - - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = val; - g->m->ptrarg[1] = &ns; - - fn = futexsleep; - runtime·onM(&fn); -} - -static void -futexsleep(void) -{ - uint32 *addr; - uint32 val; - int64 ns; - int32 ret; - Timespec ts; - - addr = g->m->ptrarg[0]; - val = g->m->scalararg[0]; - ns = *(int64*)g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->scalararg[0] = 0; - g->m->ptrarg[1] = nil; - - if(ns < 0) { - ret = runtime·sys_umtx_op(addr, UMTX_OP_WAIT_UINT_PRIVATE, val, nil, nil); - if(ret >= 0 || ret == -EINTR) - return; - goto fail; - } - // NOTE: tv_nsec is int64 on amd64, so this assumes a little-endian system. - ts.tv_nsec = 0; - ts.tv_sec = runtime·timediv(ns, 1000000000, (int32*)&ts.tv_nsec); - ret = runtime·sys_umtx_op(addr, UMTX_OP_WAIT_UINT_PRIVATE, val, nil, &ts); - if(ret >= 0 || ret == -EINTR) - return; - -fail: - runtime·prints("umtx_wait addr="); - runtime·printpointer(addr); - runtime·prints(" val="); - runtime·printint(val); - runtime·prints(" ret="); - runtime·printint(ret); - runtime·prints("\n"); - *(int32*)0x1005 = 0x1005; -} - -static void badfutexwakeup(void); - -#pragma textflag NOSPLIT -void -runtime·futexwakeup(uint32 *addr, uint32 cnt) -{ - int32 ret; - void (*fn)(void); - - ret = runtime·sys_umtx_op(addr, UMTX_OP_WAKE_PRIVATE, cnt, nil, nil); - if(ret >= 0) - return; - - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = ret; - fn = badfutexwakeup; - if(g == g->m->gsignal) - fn(); - else - runtime·onM(&fn); - *(int32*)0x1006 = 0x1006; -} - -static void -badfutexwakeup(void) -{ - void *addr; - int32 ret; - - addr = g->m->ptrarg[0]; - ret = g->m->scalararg[0]; - runtime·printf("umtx_wake addr=%p ret=%d\n", addr, ret); -} - -void runtime·thr_start(void*); - -void -runtime·newosproc(M *mp, void *stk) -{ - ThrParam param; - Sigset oset; - - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, mp->id, (int32)mp->tls[0], &mp); - } - - runtime·sigprocmask(&sigset_all, &oset); - runtime·memclr((byte*)¶m, sizeof param); - - param.start_func = runtime·thr_start; - param.arg = (byte*)mp; - - // NOTE(rsc): This code is confused. stackbase is the top of the stack - // and is equal to stk. However, it's working, so I'm not changing it. - param.stack_base = (void*)mp->g0->stack.hi; - param.stack_size = (byte*)stk - (byte*)mp->g0->stack.hi; - - param.child_tid = (void*)&mp->procid; - param.parent_tid = nil; - param.tls_base = (void*)&mp->tls[0]; - param.tls_size = sizeof mp->tls; - - mp->tls[0] = mp->id; // so 386 asm can find it - - runtime·thr_new(¶m, sizeof param); - runtime·sigprocmask(&oset, nil); -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getncpu(); -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·sigprocmask(&sigset_none, nil); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - Rlimit rl; - extern byte runtime·text[], runtime·end[]; - uintptr used; - - if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) - return 0; - if(rl.rlim_cur >= 0x7fffffff) - return 0; - - // Estimate our VM footprint excluding the heap. - // Not an exact science: use size of binary plus - // some room for thread stacks. - used = runtime·end - runtime·text + (64<<20); - if(used >= rl.rlim_cur) - return 0; - - // If there's not at least 16 MB left, we're probably - // not going to be able to do much. Treat as no limit. - rl.rlim_cur -= used; - if(rl.rlim_cur < (16<<20)) - return 0; - - return rl.rlim_cur - used; -} - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - int32 sa_flags; /* see signal options below */ - Sigset sa_mask; /* signal mask to apply */ -} SigactionT; - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__bits[0] = ~(uint32)0; - sa.sa_mask.__bits[1] = ~(uint32)0; - sa.sa_mask.__bits[2] = ~(uint32)0; - sa.sa_mask.__bits[3] = ~(uint32)0; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if((void*)sa.__sigaction_u.__sa_sigaction == runtime·sigtramp) - return runtime·sighandler; - return (void*)sa.__sigaction_u.__sa_sigaction; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(&sigset_none, nil); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_freebsd.go b/src/runtime/os_freebsd.go index 59708049c..998fbca0f 100644 --- a/src/runtime/os_freebsd.go +++ b/src/runtime/os_freebsd.go @@ -6,12 +6,29 @@ package runtime import "unsafe" -func thr_new(param unsafe.Pointer, size int32) -func sigaltstack(new, old unsafe.Pointer) -func sigaction(sig int32, new, old unsafe.Pointer) -func sigprocmask(new, old unsafe.Pointer) -func setitimer(mode int32, new, old unsafe.Pointer) +//go:noescape +func thr_new(param *thrparam, size int32) + +//go:noescape +func sigaltstack(new, old *stackt) + +//go:noescape +func sigaction(sig int32, new, old *sigactiont) + +//go:noescape +func sigprocmask(new, old *sigset) + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +//go:noescape func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 + +//go:noescape func getrlimit(kind int32, limit unsafe.Pointer) int32 func raise(sig int32) -func sys_umtx_op(addr unsafe.Pointer, mode int32, val uint32, ptr2, ts unsafe.Pointer) int32 + +//go:noescape +func sys_umtx_op(addr *uint32, mode int32, val uint32, ptr2, ts *timespec) int32 + +func osyield() diff --git a/src/runtime/os_freebsd.h b/src/runtime/os_freebsd.h deleted file mode 100644 index b86bb393c..000000000 --- a/src/runtime/os_freebsd.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -typedef byte* kevent_udata; - -int32 runtime·thr_new(ThrParam*, int32); -void runtime·sigpanic(void); -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -struct sigaction; -void runtime·sigaction(int32, struct sigaction*, struct sigaction*); -void runtime·sigprocmask(Sigset *, Sigset *); -void runtime·unblocksignals(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - -enum { - SS_DISABLE = 4, - NSIG = 33, - SI_USER = 0x10001, - RLIMIT_AS = 10, -}; - -typedef struct Rlimit Rlimit; -struct Rlimit { - int64 rlim_cur; - int64 rlim_max; -}; -int32 runtime·getrlimit(int32, Rlimit*); diff --git a/src/runtime/os_freebsd_arm.c b/src/runtime/os_freebsd_arm.go index 2f2d7767f..e049cbf9a 100644 --- a/src/runtime/os_freebsd_arm.c +++ b/src/runtime/os_freebsd_arm.go @@ -2,23 +2,16 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" +package runtime -void -runtime·checkgoarm(void) -{ +func checkgoarm() { // TODO(minux) } -#pragma textflag NOSPLIT -int64 -runtime·cputicks(void) -{ +//go:nosplit +func cputicks() int64 { // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. // TODO: need more entropy to better seed fastrand1. - return runtime·nanotime(); + return nanotime() } diff --git a/src/runtime/os_linux.c b/src/runtime/os_linux.c deleted file mode 100644 index 0d8ffc995..000000000 --- a/src/runtime/os_linux.c +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; - -static Sigset sigset_none; -static Sigset sigset_all = { ~(uint32)0, ~(uint32)0 }; - -// Linux futex. -// -// futexsleep(uint32 *addr, uint32 val) -// futexwakeup(uint32 *addr) -// -// Futexsleep atomically checks if *addr == val and if so, sleeps on addr. -// Futexwakeup wakes up threads sleeping on addr. -// Futexsleep is allowed to wake up spuriously. - -enum -{ - FUTEX_WAIT = 0, - FUTEX_WAKE = 1, -}; - -// Atomically, -// if(*addr == val) sleep -// Might be woken up spuriously; that's allowed. -// Don't sleep longer than ns; ns < 0 means forever. -#pragma textflag NOSPLIT -void -runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) -{ - Timespec ts; - - // Some Linux kernels have a bug where futex of - // FUTEX_WAIT returns an internal error code - // as an errno. Libpthread ignores the return value - // here, and so can we: as it says a few lines up, - // spurious wakeups are allowed. - - if(ns < 0) { - runtime·futex(addr, FUTEX_WAIT, val, nil, nil, 0); - return; - } - // NOTE: tv_nsec is int64 on amd64, so this assumes a little-endian system. - ts.tv_nsec = 0; - ts.tv_sec = runtime·timediv(ns, 1000000000LL, (int32*)&ts.tv_nsec); - runtime·futex(addr, FUTEX_WAIT, val, &ts, nil, 0); -} - -static void badfutexwakeup(void); - -// If any procs are sleeping on addr, wake up at most cnt. -#pragma textflag NOSPLIT -void -runtime·futexwakeup(uint32 *addr, uint32 cnt) -{ - int64 ret; - void (*fn)(void); - - ret = runtime·futex(addr, FUTEX_WAKE, cnt, nil, nil, 0); - if(ret >= 0) - return; - - // I don't know that futex wakeup can return - // EAGAIN or EINTR, but if it does, it would be - // safe to loop and call futex again. - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = (int32)ret; // truncated but fine - fn = badfutexwakeup; - if(g == g->m->gsignal) - fn(); - else - runtime·onM(&fn); - *(int32*)0x1006 = 0x1006; -} - -static void -badfutexwakeup(void) -{ - void *addr; - int64 ret; - - addr = g->m->ptrarg[0]; - ret = (int32)g->m->scalararg[0]; - runtime·printf("futexwakeup addr=%p returned %D\n", addr, ret); -} - -extern runtime·sched_getaffinity(uintptr pid, uintptr len, uintptr *buf); -static int32 -getproccount(void) -{ - uintptr buf[16], t; - int32 r, cnt, i; - - cnt = 0; - r = runtime·sched_getaffinity(0, sizeof(buf), buf); - if(r > 0) - for(i = 0; i < r/sizeof(buf[0]); i++) { - t = buf[i]; - t = t - ((t >> 1) & 0x5555555555555555ULL); - t = (t & 0x3333333333333333ULL) + ((t >> 2) & 0x3333333333333333ULL); - cnt += (int32)((((t + (t >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56); - } - - return cnt ? cnt : 1; -} - -// Clone, the Linux rfork. -enum -{ - CLONE_VM = 0x100, - CLONE_FS = 0x200, - CLONE_FILES = 0x400, - CLONE_SIGHAND = 0x800, - CLONE_PTRACE = 0x2000, - CLONE_VFORK = 0x4000, - CLONE_PARENT = 0x8000, - CLONE_THREAD = 0x10000, - CLONE_NEWNS = 0x20000, - CLONE_SYSVSEM = 0x40000, - CLONE_SETTLS = 0x80000, - CLONE_PARENT_SETTID = 0x100000, - CLONE_CHILD_CLEARTID = 0x200000, - CLONE_UNTRACED = 0x800000, - CLONE_CHILD_SETTID = 0x1000000, - CLONE_STOPPED = 0x2000000, - CLONE_NEWUTS = 0x4000000, - CLONE_NEWIPC = 0x8000000, -}; - -void -runtime·newosproc(M *mp, void *stk) -{ - int32 ret; - int32 flags; - Sigset oset; - - /* - * note: strace gets confused if we use CLONE_PTRACE here. - */ - flags = CLONE_VM /* share memory */ - | CLONE_FS /* share cwd, etc */ - | CLONE_FILES /* share fd table */ - | CLONE_SIGHAND /* share sig handler table */ - | CLONE_THREAD /* revisit - okay for now */ - ; - - mp->tls[0] = mp->id; // so 386 asm can find it - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p clone=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, runtime·clone, mp->id, (int32)mp->tls[0], &mp); - } - - // Disable signals during clone, so that the new thread starts - // with signals disabled. It will enable them in minit. - runtime·rtsigprocmask(SIG_SETMASK, &sigset_all, &oset, sizeof oset); - ret = runtime·clone(flags, stk, mp, mp->g0, runtime·mstart); - runtime·rtsigprocmask(SIG_SETMASK, &oset, nil, sizeof oset); - - if(ret < 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -ret); - runtime·throw("runtime.newosproc"); - } -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getproccount(); -} - -// Random bytes initialized at startup. These come -// from the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.c). -byte* runtime·startup_random_data; -uint32 runtime·startup_random_data_len; - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - if(runtime·startup_random_data != nil) { - *rnd = runtime·startup_random_data; - *rnd_len = runtime·startup_random_data_len; - } else { - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); - } -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling. - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·rtsigprocmask(SIG_SETMASK, &sigset_none, nil, sizeof(Sigset)); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - Rlimit rl; - extern byte runtime·text[], runtime·end[]; - uintptr used; - - if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) - return 0; - if(rl.rlim_cur >= 0x7fffffff) - return 0; - - // Estimate our VM footprint excluding the heap. - // Not an exact science: use size of binary plus - // some room for thread stacks. - used = runtime·end - runtime·text + (64<<20); - if(used >= rl.rlim_cur) - return 0; - - // If there's not at least 16 MB left, we're probably - // not going to be able to do much. Treat as no limit. - rl.rlim_cur -= used; - if(rl.rlim_cur < (16<<20)) - return 0; - - return rl.rlim_cur - used; -} - -#ifdef GOARCH_386 -#define sa_handler k_sa_handler -#endif - -/* - * This assembler routine takes the args from registers, puts them on the stack, - * and calls sighandler(). - */ -extern void runtime·sigtramp(void); -extern void runtime·sigreturn(void); // calls rt_sigreturn, only used with SA_RESTORER - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - // Although Linux manpage says "sa_restorer element is obsolete and - // should not be used". x86_64 kernel requires it. Only use it on - // x86. -#ifdef GOARCH_386 - sa.sa_restorer = (void*)runtime·sigreturn; -#endif -#ifdef GOARCH_amd64 - sa.sa_restorer = (void*)runtime·sigreturn; -#endif - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.sa_handler = fn; - if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction failure"); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction read failure"); - if((void*)sa.sa_handler == runtime·sigtramp) - return runtime·sighandler; - return (void*)sa.sa_handler; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - SigaltstackT st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·rtsigprocmask(SIG_SETMASK, &sigset_none, nil, sizeof sigset_none); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 41123ad57..113219aab 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -6,12 +6,28 @@ package runtime import "unsafe" +//go:noescape func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 + +//go:noescape func clone(flags int32, stk, mm, gg, fn unsafe.Pointer) int32 -func rt_sigaction(sig uintptr, new, old unsafe.Pointer, size uintptr) int32 -func sigaltstack(new, old unsafe.Pointer) -func setitimer(mode int32, new, old unsafe.Pointer) -func rtsigprocmask(sig int32, new, old unsafe.Pointer, size int32) + +//go:noescape +func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 + +//go:noescape +func sigaltstack(new, old *sigaltstackt) + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +//go:noescape +func rtsigprocmask(sig uint32, new, old *sigset, size int32) + +//go:noescape func getrlimit(kind int32, limit unsafe.Pointer) int32 -func raise(sig int32) +func raise(sig uint32) + +//go:noescape func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 +func osyield() diff --git a/src/runtime/os_linux.h b/src/runtime/os_linux.h deleted file mode 100644 index 75606d615..000000000 --- a/src/runtime/os_linux.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - - -// Linux-specific system calls -int32 runtime·futex(uint32*, int32, uint32, Timespec*, uint32*, uint32); -int32 runtime·clone(int32, void*, M*, G*, void(*)(void)); - -struct SigactionT; -int32 runtime·rt_sigaction(uintptr, struct SigactionT*, void*, uintptr); - -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -void runtime·sigpanic(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); - -enum { - SS_DISABLE = 2, - NSIG = 65, - SI_USER = 0, - SIG_SETMASK = 2, - RLIMIT_AS = 9, -}; - -// It's hard to tease out exactly how big a Sigset is, but -// rt_sigprocmask crashes if we get it wrong, so if binaries -// are running, this is right. -typedef struct Sigset Sigset; -struct Sigset -{ - uint32 mask[2]; -}; -void runtime·rtsigprocmask(int32, Sigset*, Sigset*, int32); -void runtime·unblocksignals(void); - -typedef struct Rlimit Rlimit; -struct Rlimit { - uintptr rlim_cur; - uintptr rlim_max; -}; -int32 runtime·getrlimit(int32, Rlimit*); diff --git a/src/runtime/os_linux_386.c b/src/runtime/os_linux_386.c deleted file mode 100644 index dc89d04e2..000000000 --- a/src/runtime/os_linux_386.c +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" - -#define AT_NULL 0 -#define AT_RANDOM 25 -#define AT_SYSINFO 32 -extern uint32 runtime·_vdso; - -#pragma textflag NOSPLIT -void -runtime·linux_setup_vdso(int32 argc, byte **argv) -{ - byte **envp; - uint32 *auxv; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - if(auxv[0] == AT_SYSINFO) { - runtime·_vdso = auxv[1]; - continue; - } - if(auxv[0] == AT_RANDOM) { - runtime·startup_random_data = (byte*)auxv[1]; - runtime·startup_random_data_len = 16; - continue; - } - } -} diff --git a/src/runtime/os_linux_386.go b/src/runtime/os_linux_386.go new file mode 100644 index 000000000..c4f95804a --- /dev/null +++ b/src/runtime/os_linux_386.go @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + _AT_NULL = 0 + _AT_RANDOM = 25 + _AT_SYSINFO = 32 +) + +var _vdso uint32 + +//go:nosplit +func linux_setup_vdso(argc int32, argv **byte) { + // skip over argv, envv to get to auxv + n := argc + 1 + for argv_index(argv, n) != nil { + n++ + } + n++ + auxv := (*[1 << 28]uint32)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i] != _AT_NULL; i += 2 { + switch auxv[i] { + case _AT_SYSINFO: + _vdso = auxv[i+1] + + case _AT_RANDOM: + startup_random_data = (*byte)(unsafe.Pointer(uintptr(auxv[i+1]))) + startup_random_data_len = 16 + } + } +} diff --git a/src/runtime/os_linux_arm.c b/src/runtime/os_linux_arm.c deleted file mode 100644 index e3eda7c2d..000000000 --- a/src/runtime/os_linux_arm.c +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" - -#define AT_NULL 0 -#define AT_PLATFORM 15 // introduced in at least 2.6.11 -#define AT_HWCAP 16 // introduced in at least 2.6.11 -#define AT_RANDOM 25 // introduced in 2.6.29 -#define HWCAP_VFP (1 << 6) // introduced in at least 2.6.11 -#define HWCAP_VFPv3 (1 << 13) // introduced in 2.6.30 -static uint32 runtime·randomNumber; -uint8 runtime·armArch = 6; // we default to ARMv6 -uint32 runtime·hwcap; // set by setup_auxv -extern uint8 runtime·goarm; // set by 5l - -void -runtime·checkgoarm(void) -{ - if(runtime·goarm > 5 && !(runtime·hwcap & HWCAP_VFP)) { - runtime·printf("runtime: this CPU has no floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=5.\n", runtime·goarm); - runtime·exit(1); - } - if(runtime·goarm > 6 && !(runtime·hwcap & HWCAP_VFPv3)) { - runtime·printf("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=6.\n", runtime·goarm); - runtime·exit(1); - } -} - -#pragma textflag NOSPLIT -void -runtime·setup_auxv(int32 argc, byte **argv) -{ - byte **envp; - byte *rnd; - uint32 *auxv; - uint32 t; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - switch(auxv[0]) { - case AT_RANDOM: // kernel provided 16-byte worth of random data - if(auxv[1]) { - rnd = (byte*)auxv[1]; - runtime·randomNumber = rnd[4] | rnd[5]<<8 | rnd[6]<<16 | rnd[7]<<24; - } - break; - case AT_PLATFORM: // v5l, v6l, v7l - if(auxv[1]) { - t = *(uint8*)(auxv[1]+1); - if(t >= '5' && t <= '7') - runtime·armArch = t - '0'; - } - break; - case AT_HWCAP: // CPU capability bit flags - runtime·hwcap = auxv[1]; - break; - } - } -} - -#pragma textflag NOSPLIT -int64 -runtime·cputicks(void) -{ - // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - // runtime·randomNumber provides better seeding of fastrand1. - return runtime·nanotime() + runtime·randomNumber; -} diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go new file mode 100644 index 000000000..d5b37d6ab --- /dev/null +++ b/src/runtime/os_linux_arm.go @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + _AT_NULL = 0 + _AT_PLATFORM = 15 // introduced in at least 2.6.11 + _AT_HWCAP = 16 // introduced in at least 2.6.11 + _AT_RANDOM = 25 // introduced in 2.6.29 + + _HWCAP_VFP = 1 << 6 // introduced in at least 2.6.11 + _HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30 +) + +var randomNumber uint32 +var armArch uint8 = 6 // we default to ARMv6 +var hwcap uint32 // set by setup_auxv +var goarm uint8 // set by 5l + +func checkgoarm() { + if goarm > 5 && hwcap&_HWCAP_VFP == 0 { + print("runtime: this CPU has no floating point hardware, so it cannot run\n") + print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n") + exit(1) + } + if goarm > 6 && hwcap&_HWCAP_VFPv3 == 0 { + print("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n") + print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n") + exit(1) + } +} + +//go:nosplit +func setup_auxv(argc int32, argv **byte) { + // skip over argv, envv to get to auxv + n := argc + 1 + for argv_index(argv, n) != nil { + n++ + } + n++ + auxv := (*[1 << 28]uint32)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i] != _AT_NULL; i += 2 { + switch auxv[i] { + case _AT_RANDOM: // kernel provides a pointer to 16-bytes worth of random data + if auxv[i+1] != 0 { + // the pointer provided may not be word alined, so we must to treat it + // as a byte array. + rnd := (*[16]byte)(unsafe.Pointer(uintptr(auxv[i+1]))) + randomNumber = uint32(rnd[0]) | uint32(rnd[1])<<8 | uint32(rnd[2])<<16 | uint32(rnd[3])<<24 + } + + case _AT_PLATFORM: // v5l, v6l, v7l + t := *(*uint8)(unsafe.Pointer(uintptr(auxv[i+1] + 1))) + if '5' <= t && t <= '7' { + armArch = t - '0' + } + + case _AT_HWCAP: // CPU capability bit flags + hwcap = auxv[i+1] + } + } +} + +func cputicks() int64 { + // Currently cputicks() is used in blocking profiler and to seed fastrand1(). + // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // randomNumber provides better seeding of fastrand1. + return nanotime() + int64(randomNumber) +} diff --git a/src/runtime/os_openbsd.c b/src/runtime/os_openbsd.c deleted file mode 100644 index eebaa13ee..000000000 --- a/src/runtime/os_openbsd.c +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -enum -{ - ESRCH = 3, - EAGAIN = 35, - EWOULDBLOCK = EAGAIN, - ENOTSUP = 91, - - // From OpenBSD's sys/time.h - CLOCK_REALTIME = 0, - CLOCK_VIRTUAL = 1, - CLOCK_PROF = 2, - CLOCK_MONOTONIC = 3 -}; - -extern SigTab runtime·sigtab[]; - -static Sigset sigset_none; -static Sigset sigset_all = ~(Sigset)0; - -extern int32 runtime·tfork(TforkT *param, uintptr psize, M *mp, G *gp, void (*fn)(void)); -extern int32 runtime·thrsleep(void *ident, int32 clock_id, void *tsp, void *lock, const int32 *abort); -extern int32 runtime·thrwakeup(void *ident, int32 n); - -// From OpenBSD's <sys/sysctl.h> -#define CTL_HW 6 -#define HW_NCPU 3 - -static int32 -getncpu(void) -{ - uint32 mib[2]; - uint32 out; - int32 ret; - uintptr nout; - - // Fetch hw.ncpu via sysctl. - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - nout = sizeof out; - out = 0; - ret = runtime·sysctl(mib, 2, (byte*)&out, &nout, nil, 0); - if(ret >= 0) - return out; - else - return 1; -} - -#pragma textflag NOSPLIT -uintptr -runtime·semacreate(void) -{ - return 1; -} - -#pragma textflag NOSPLIT -int32 -runtime·semasleep(int64 ns) -{ - Timespec ts, *tsp = nil; - - // Compute sleep deadline. - if(ns >= 0) { - int32 nsec; - ns += runtime·nanotime(); - ts.tv_sec = runtime·timediv(ns, 1000000000, &nsec); - ts.tv_nsec = nsec; // tv_nsec is int64 on amd64 - tsp = &ts; - } - - for(;;) { - int32 ret; - - // spin-mutex lock - while(runtime·xchg(&g->m->waitsemalock, 1)) - runtime·osyield(); - - if(g->m->waitsemacount != 0) { - // semaphore is available. - g->m->waitsemacount--; - // spin-mutex unlock - runtime·atomicstore(&g->m->waitsemalock, 0); - return 0; // semaphore acquired - } - - // sleep until semaphore != 0 or timeout. - // thrsleep unlocks m->waitsemalock. - ret = runtime·thrsleep(&g->m->waitsemacount, CLOCK_MONOTONIC, tsp, &g->m->waitsemalock, (int32 *)&g->m->waitsemacount); - if(ret == EWOULDBLOCK) - return -1; - } -} - -static void badsemawakeup(void); - -#pragma textflag NOSPLIT -void -runtime·semawakeup(M *mp) -{ - uint32 ret; - void *oldptr; - uint32 oldscalar; - void (*fn)(void); - - // spin-mutex lock - while(runtime·xchg(&mp->waitsemalock, 1)) - runtime·osyield(); - mp->waitsemacount++; - ret = runtime·thrwakeup(&mp->waitsemacount, 1); - if(ret != 0 && ret != ESRCH) { - // semawakeup can be called on signal stack. - // Save old ptrarg/scalararg so we can restore them. - oldptr = g->m->ptrarg[0]; - oldscalar = g->m->scalararg[0]; - g->m->ptrarg[0] = mp; - g->m->scalararg[0] = ret; - fn = badsemawakeup; - if(g == g->m->gsignal) - fn(); - else - runtime·onM(&fn); - g->m->ptrarg[0] = oldptr; - g->m->scalararg[0] = oldscalar; - } - // spin-mutex unlock - runtime·atomicstore(&mp->waitsemalock, 0); -} - -static void -badsemawakeup(void) -{ - M *mp; - int32 ret; - - mp = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - ret = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - - runtime·printf("thrwakeup addr=%p sem=%d ret=%d\n", &mp->waitsemacount, mp->waitsemacount, ret); -} - -void -runtime·newosproc(M *mp, void *stk) -{ - TforkT param; - Sigset oset; - int32 ret; - - if(0) { - runtime·printf( - "newosproc stk=%p m=%p g=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, mp->id, (int32)mp->tls[0], &mp); - } - - mp->tls[0] = mp->id; // so 386 asm can find it - - param.tf_tcb = (byte*)&mp->tls[0]; - param.tf_tid = (int32*)&mp->procid; - param.tf_stack = stk; - - oset = runtime·sigprocmask(SIG_SETMASK, sigset_all); - ret = runtime·tfork(¶m, sizeof(param), mp, mp->g0, runtime·mstart); - runtime·sigprocmask(SIG_SETMASK, oset); - - if(ret < 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount() - 1, -ret); - if (ret == -ENOTSUP) - runtime·printf("runtime: is kern.rthreads disabled?\n"); - runtime·throw("runtime.newosproc"); - } -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getncpu(); -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·sigprocmask(SIG_SETMASK, sigset_none); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - return 0; -} - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - uint32 sa_mask; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} SigactionT; - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0U; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if((void*)sa.__sigaction_u.__sa_sigaction == runtime·sigtramp) - return runtime·sighandler; - return (void*)sa.__sigaction_u.__sa_sigaction; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(SIG_SETMASK, sigset_none); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_openbsd.go b/src/runtime/os_openbsd.go index a000f963e..9e5adcd3d 100644 --- a/src/runtime/os_openbsd.go +++ b/src/runtime/os_openbsd.go @@ -4,14 +4,30 @@ package runtime -import "unsafe" +//go:noescape +func setitimer(mode int32, new, old *itimerval) -func setitimer(mode int32, new, old unsafe.Pointer) -func sigaction(sig int32, new, old unsafe.Pointer) -func sigaltstack(new, old unsafe.Pointer) +//go:noescape +func sigaction(sig int32, new, old *sigactiont) + +//go:noescape +func sigaltstack(new, old *stackt) + +//go:noescape func sigprocmask(mode int32, new uint32) uint32 + +//go:noescape func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 + func raise(sig int32) -func tfork(param unsafe.Pointer, psize uintptr, mm, gg, fn unsafe.Pointer) int32 -func thrsleep(ident unsafe.Pointer, clock_id int32, tsp, lock, abort unsafe.Pointer) int32 -func thrwakeup(ident unsafe.Pointer, n int32) int32 + +//go:noescape +func tfork(param *tforkt, psize uintptr, mm *m, gg *g, fn uintptr) int32 + +//go:noescape +func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *int32) int32 + +//go:noescape +func thrwakeup(ident uintptr, n int32) int32 + +func osyield() diff --git a/src/runtime/os_openbsd.h b/src/runtime/os_openbsd.h deleted file mode 100644 index 6ad98109e..000000000 --- a/src/runtime/os_openbsd.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - - -typedef byte* kevent_udata; - -struct sigaction; - -void runtime·sigpanic(void); - -void runtime·setitimer(int32, Itimerval*, Itimerval*); -void runtime·sigaction(int32, struct sigaction*, struct sigaction*); -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -Sigset runtime·sigprocmask(int32, Sigset); -void runtime·unblocksignals(void); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - -enum { - SS_DISABLE = 4, - SIG_BLOCK = 1, - SIG_UNBLOCK = 2, - SIG_SETMASK = 3, - NSIG = 33, - SI_USER = 0, -}; diff --git a/src/runtime/os_solaris.c b/src/runtime/os_solaris.c deleted file mode 100644 index e16b8e637..000000000 --- a/src/runtime/os_solaris.c +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -#pragma dynexport runtime·end _end -#pragma dynexport runtime·etext _etext -#pragma dynexport runtime·edata _edata - -#pragma dynimport libc·___errno ___errno "libc.so" -#pragma dynimport libc·clock_gettime clock_gettime "libc.so" -#pragma dynimport libc·close close "libc.so" -#pragma dynimport libc·exit exit "libc.so" -#pragma dynimport libc·fstat fstat "libc.so" -#pragma dynimport libc·getcontext getcontext "libc.so" -#pragma dynimport libc·getrlimit getrlimit "libc.so" -#pragma dynimport libc·malloc malloc "libc.so" -#pragma dynimport libc·mmap mmap "libc.so" -#pragma dynimport libc·munmap munmap "libc.so" -#pragma dynimport libc·open open "libc.so" -#pragma dynimport libc·pthread_attr_destroy pthread_attr_destroy "libc.so" -#pragma dynimport libc·pthread_attr_getstack pthread_attr_getstack "libc.so" -#pragma dynimport libc·pthread_attr_init pthread_attr_init "libc.so" -#pragma dynimport libc·pthread_attr_setdetachstate pthread_attr_setdetachstate "libc.so" -#pragma dynimport libc·pthread_attr_setstack pthread_attr_setstack "libc.so" -#pragma dynimport libc·pthread_create pthread_create "libc.so" -#pragma dynimport libc·raise raise "libc.so" -#pragma dynimport libc·read read "libc.so" -#pragma dynimport libc·select select "libc.so" -#pragma dynimport libc·sched_yield sched_yield "libc.so" -#pragma dynimport libc·sem_init sem_init "libc.so" -#pragma dynimport libc·sem_post sem_post "libc.so" -#pragma dynimport libc·sem_reltimedwait_np sem_reltimedwait_np "libc.so" -#pragma dynimport libc·sem_wait sem_wait "libc.so" -#pragma dynimport libc·setitimer setitimer "libc.so" -#pragma dynimport libc·sigaction sigaction "libc.so" -#pragma dynimport libc·sigaltstack sigaltstack "libc.so" -#pragma dynimport libc·sigprocmask sigprocmask "libc.so" -#pragma dynimport libc·sysconf sysconf "libc.so" -#pragma dynimport libc·usleep usleep "libc.so" -#pragma dynimport libc·write write "libc.so" - -extern uintptr libc·___errno; -extern uintptr libc·clock_gettime; -extern uintptr libc·close; -extern uintptr libc·exit; -extern uintptr libc·fstat; -extern uintptr libc·getcontext; -extern uintptr libc·getrlimit; -extern uintptr libc·malloc; -extern uintptr libc·mmap; -extern uintptr libc·munmap; -extern uintptr libc·open; -extern uintptr libc·pthread_attr_destroy; -extern uintptr libc·pthread_attr_getstack; -extern uintptr libc·pthread_attr_init; -extern uintptr libc·pthread_attr_setdetachstate; -extern uintptr libc·pthread_attr_setstack; -extern uintptr libc·pthread_create; -extern uintptr libc·raise; -extern uintptr libc·read; -extern uintptr libc·sched_yield; -extern uintptr libc·select; -extern uintptr libc·sem_init; -extern uintptr libc·sem_post; -extern uintptr libc·sem_reltimedwait_np; -extern uintptr libc·sem_wait; -extern uintptr libc·setitimer; -extern uintptr libc·sigaction; -extern uintptr libc·sigaltstack; -extern uintptr libc·sigprocmask; -extern uintptr libc·sysconf; -extern uintptr libc·usleep; -extern uintptr libc·write; - -void runtime·getcontext(Ucontext *context); -int32 runtime·pthread_attr_destroy(PthreadAttr* attr); -int32 runtime·pthread_attr_init(PthreadAttr* attr); -int32 runtime·pthread_attr_getstack(PthreadAttr* attr, void** addr, uint64* size); -int32 runtime·pthread_attr_setdetachstate(PthreadAttr* attr, int32 state); -int32 runtime·pthread_attr_setstack(PthreadAttr* attr, void* addr, uint64 size); -int32 runtime·pthread_create(Pthread* thread, PthreadAttr* attr, void(*fn)(void), void *arg); -uint32 runtime·tstart_sysvicall(M *newm); -int32 runtime·sem_init(SemT* sem, int32 pshared, uint32 value); -int32 runtime·sem_post(SemT* sem); -int32 runtime·sem_reltimedwait_np(SemT* sem, Timespec* timeout); -int32 runtime·sem_wait(SemT* sem); -int64 runtime·sysconf(int32 name); - -extern SigTab runtime·sigtab[]; -static Sigset sigset_none; -static Sigset sigset_all = { ~(uint32)0, ~(uint32)0, ~(uint32)0, ~(uint32)0, }; - -static int32 -getncpu(void) -{ - int32 n; - - n = (int32)runtime·sysconf(_SC_NPROCESSORS_ONLN); - if(n < 1) - return 1; - return n; -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getncpu(); -} - -void -runtime·newosproc(M *mp, void *stk) -{ - PthreadAttr attr; - Sigset oset; - Pthread tid; - int32 ret; - uint64 size; - - USED(stk); - if(runtime·pthread_attr_init(&attr) != 0) - runtime·throw("pthread_attr_init"); - if(runtime·pthread_attr_setstack(&attr, 0, 0x200000) != 0) - runtime·throw("pthread_attr_setstack"); - size = 0; - if(runtime·pthread_attr_getstack(&attr, (void**)&mp->g0->stack.hi, &size) != 0) - runtime·throw("pthread_attr_getstack"); - mp->g0->stack.lo = mp->g0->stack.hi - size; - if(runtime·pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) - runtime·throw("pthread_attr_setdetachstate"); - - // Disable signals during create, so that the new thread starts - // with signals disabled. It will enable them in minit. - runtime·sigprocmask(SIG_SETMASK, &sigset_all, &oset); - ret = runtime·pthread_create(&tid, &attr, (void (*)(void))runtime·tstart_sysvicall, mp); - runtime·sigprocmask(SIG_SETMASK, &oset, nil); - if(ret != 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), ret); - runtime·throw("runtime.newosproc"); - } -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - runtime·asmcgocall(runtime·miniterrno, (void *)libc·___errno); - // Initialize signal handling - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - Rlimit rl; - extern byte runtime·text[], runtime·end[]; - uintptr used; - - if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) - return 0; - if(rl.rlim_cur >= 0x7fffffff) - return 0; - - // Estimate our VM footprint excluding the heap. - // Not an exact science: use size of binary plus - // some room for thread stacks. - used = runtime·end - runtime·text + (64<<20); - if(used >= rl.rlim_cur) - return 0; - - // If there's not at least 16 MB left, we're probably - // not going to be able to do much. Treat as no limit. - rl.rlim_cur -= used; - if(rl.rlim_cur < (16<<20)) - return 0; - - return rl.rlim_cur - used; -} - -void -runtime·setprof(bool on) -{ - USED(on); -} - -extern void runtime·sigtramp(void); - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__sigbits[0] = ~(uint32)0; - sa.sa_mask.__sigbits[1] = ~(uint32)0; - sa.sa_mask.__sigbits[2] = ~(uint32)0; - sa.sa_mask.__sigbits[3] = ~(uint32)0; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - *((void**)&sa._funcptr[0]) = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(*((void**)&sa._funcptr[0]) == runtime·sigtramp) - return runtime·sighandler; - return *((void**)&sa._funcptr[0]); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -#pragma textflag NOSPLIT -uintptr -runtime·semacreate(void) -{ - SemT* sem; - - // Call libc's malloc rather than runtime·malloc. This will - // allocate space on the C heap. We can't call runtime·malloc - // here because it could cause a deadlock. - g->m->libcall.fn = (uintptr)(void*)libc·malloc; - g->m->libcall.n = 1; - runtime·memclr((byte*)&g->m->scratch, sizeof(g->m->scratch)); - g->m->scratch.v[0] = (uintptr)sizeof(*sem); - g->m->libcall.args = (uintptr)(uintptr*)&g->m->scratch; - runtime·asmcgocall(runtime·asmsysvicall6, &g->m->libcall); - sem = (void*)g->m->libcall.r1; - if(runtime·sem_init(sem, 0, 0) != 0) - runtime·throw("sem_init"); - return (uintptr)sem; -} - -#pragma textflag NOSPLIT -int32 -runtime·semasleep(int64 ns) -{ - M *m; - - m = g->m; - if(ns >= 0) { - m->ts.tv_sec = ns / 1000000000LL; - m->ts.tv_nsec = ns % 1000000000LL; - - m->libcall.fn = (uintptr)(void*)libc·sem_reltimedwait_np; - m->libcall.n = 2; - runtime·memclr((byte*)&m->scratch, sizeof(m->scratch)); - m->scratch.v[0] = m->waitsema; - m->scratch.v[1] = (uintptr)&m->ts; - m->libcall.args = (uintptr)(uintptr*)&m->scratch; - runtime·asmcgocall(runtime·asmsysvicall6, &m->libcall); - if(*m->perrno != 0) { - if(*m->perrno == ETIMEDOUT || *m->perrno == EAGAIN || *m->perrno == EINTR) - return -1; - runtime·throw("sem_reltimedwait_np"); - } - return 0; - } - for(;;) { - m->libcall.fn = (uintptr)(void*)libc·sem_wait; - m->libcall.n = 1; - runtime·memclr((byte*)&m->scratch, sizeof(m->scratch)); - m->scratch.v[0] = m->waitsema; - m->libcall.args = (uintptr)(uintptr*)&m->scratch; - runtime·asmcgocall(runtime·asmsysvicall6, &m->libcall); - if(m->libcall.r1 == 0) - break; - if(*m->perrno == EINTR) - continue; - runtime·throw("sem_wait"); - } - return 0; -} - -#pragma textflag NOSPLIT -void -runtime·semawakeup(M *mp) -{ - SemT* sem = (SemT*)mp->waitsema; - if(runtime·sem_post(sem) != 0) - runtime·throw("sem_post"); -} - -#pragma textflag NOSPLIT -int32 -runtime·close(int32 fd) -{ - return runtime·sysvicall1(libc·close, (uintptr)fd); -} - -#pragma textflag NOSPLIT -void -runtime·exit(int32 r) -{ - runtime·sysvicall1(libc·exit, (uintptr)r); -} - -#pragma textflag NOSPLIT -/* int32 */ void -runtime·getcontext(Ucontext* context) -{ - runtime·sysvicall1(libc·getcontext, (uintptr)context); -} - -#pragma textflag NOSPLIT -int32 -runtime·getrlimit(int32 res, Rlimit* rlp) -{ - return runtime·sysvicall2(libc·getrlimit, (uintptr)res, (uintptr)rlp); -} - -#pragma textflag NOSPLIT -uint8* -runtime·mmap(byte* addr, uintptr len, int32 prot, int32 flags, int32 fildes, uint32 off) -{ - return (uint8*)runtime·sysvicall6(libc·mmap, (uintptr)addr, (uintptr)len, (uintptr)prot, (uintptr)flags, (uintptr)fildes, (uintptr)off); -} - -#pragma textflag NOSPLIT -void -runtime·munmap(byte* addr, uintptr len) -{ - runtime·sysvicall2(libc·munmap, (uintptr)addr, (uintptr)len); -} - -extern int64 runtime·nanotime1(void); -#pragma textflag NOSPLIT -int64 -runtime·nanotime(void) -{ - return runtime·sysvicall0((uintptr)runtime·nanotime1); -} - -#pragma textflag NOSPLIT -int32 -runtime·open(int8* path, int32 oflag, int32 mode) -{ - return runtime·sysvicall3(libc·open, (uintptr)path, (uintptr)oflag, (uintptr)mode); -} - -int32 -runtime·pthread_attr_destroy(PthreadAttr* attr) -{ - return runtime·sysvicall1(libc·pthread_attr_destroy, (uintptr)attr); -} - -int32 -runtime·pthread_attr_getstack(PthreadAttr* attr, void** addr, uint64* size) -{ - return runtime·sysvicall3(libc·pthread_attr_getstack, (uintptr)attr, (uintptr)addr, (uintptr)size); -} - -int32 -runtime·pthread_attr_init(PthreadAttr* attr) -{ - return runtime·sysvicall1(libc·pthread_attr_init, (uintptr)attr); -} - -int32 -runtime·pthread_attr_setdetachstate(PthreadAttr* attr, int32 state) -{ - return runtime·sysvicall2(libc·pthread_attr_setdetachstate, (uintptr)attr, (uintptr)state); -} - -int32 -runtime·pthread_attr_setstack(PthreadAttr* attr, void* addr, uint64 size) -{ - return runtime·sysvicall3(libc·pthread_attr_setstack, (uintptr)attr, (uintptr)addr, (uintptr)size); -} - -int32 -runtime·pthread_create(Pthread* thread, PthreadAttr* attr, void(*fn)(void), void *arg) -{ - return runtime·sysvicall4(libc·pthread_create, (uintptr)thread, (uintptr)attr, (uintptr)fn, (uintptr)arg); -} - -/* int32 */ void -runtime·raise(int32 sig) -{ - runtime·sysvicall1(libc·raise, (uintptr)sig); -} - -#pragma textflag NOSPLIT -int32 -runtime·read(int32 fd, void* buf, int32 nbyte) -{ - return runtime·sysvicall3(libc·read, (uintptr)fd, (uintptr)buf, (uintptr)nbyte); -} - -#pragma textflag NOSPLIT -int32 -runtime·sem_init(SemT* sem, int32 pshared, uint32 value) -{ - return runtime·sysvicall3(libc·sem_init, (uintptr)sem, (uintptr)pshared, (uintptr)value); -} - -#pragma textflag NOSPLIT -int32 -runtime·sem_post(SemT* sem) -{ - return runtime·sysvicall1(libc·sem_post, (uintptr)sem); -} - -#pragma textflag NOSPLIT -int32 -runtime·sem_reltimedwait_np(SemT* sem, Timespec* timeout) -{ - return runtime·sysvicall2(libc·sem_reltimedwait_np, (uintptr)sem, (uintptr)timeout); -} - -#pragma textflag NOSPLIT -int32 -runtime·sem_wait(SemT* sem) -{ - return runtime·sysvicall1(libc·sem_wait, (uintptr)sem); -} - -/* int32 */ void -runtime·setitimer(int32 which, Itimerval* value, Itimerval* ovalue) -{ - runtime·sysvicall3(libc·setitimer, (uintptr)which, (uintptr)value, (uintptr)ovalue); -} - -/* int32 */ void -runtime·sigaction(int32 sig, struct SigactionT* act, struct SigactionT* oact) -{ - runtime·sysvicall3(libc·sigaction, (uintptr)sig, (uintptr)act, (uintptr)oact); -} - -/* int32 */ void -runtime·sigaltstack(SigaltstackT* ss, SigaltstackT* oss) -{ - runtime·sysvicall2(libc·sigaltstack, (uintptr)ss, (uintptr)oss); -} - -/* int32 */ void -runtime·sigprocmask(int32 how, Sigset* set, Sigset* oset) -{ - runtime·sysvicall3(libc·sigprocmask, (uintptr)how, (uintptr)set, (uintptr)oset); -} - -int64 -runtime·sysconf(int32 name) -{ - return runtime·sysvicall1(libc·sysconf, (uintptr)name); -} - -extern void runtime·usleep1(uint32); - -#pragma textflag NOSPLIT -void -runtime·usleep(uint32 µs) -{ - runtime·usleep1(µs); -} - -#pragma textflag NOSPLIT -int32 -runtime·write(uintptr fd, void* buf, int32 nbyte) -{ - return runtime·sysvicall3(libc·write, (uintptr)fd, (uintptr)buf, (uintptr)nbyte); -} - -extern void runtime·osyield1(void); - -#pragma textflag NOSPLIT -void -runtime·osyield(void) -{ - // Check the validity of m because we might be called in cgo callback - // path early enough where there isn't a m available yet. - if(g && g->m != nil) { - runtime·sysvicall0(libc·sched_yield); - return; - } - runtime·osyield1(); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_solaris.go b/src/runtime/os_solaris.go index ca1315120..6864ef938 100644 --- a/src/runtime/os_solaris.go +++ b/src/runtime/os_solaris.go @@ -6,53 +6,35 @@ package runtime import "unsafe" -func setitimer(mode int32, new, old unsafe.Pointer) -func sigaction(sig int32, new, old unsafe.Pointer) -func sigaltstack(new, old unsafe.Pointer) -func sigprocmask(mode int32, new, old unsafe.Pointer) -func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 -func getrlimit(kind int32, limit unsafe.Pointer) -func miniterrno(fn unsafe.Pointer) -func raise(sig int32) -func getcontext(ctxt unsafe.Pointer) -func tstart_sysvicall(mm unsafe.Pointer) uint32 -func nanotime1() int64 -func usleep1(usec uint32) -func osyield1() -func netpollinit() -func netpollopen(fd uintptr, pd *pollDesc) int32 -func netpollclose(fd uintptr) int32 -func netpollarm(pd *pollDesc, mode int) - -type libcFunc byte +type libcFunc uintptr var asmsysvicall6 libcFunc //go:nosplit -func sysvicall0(fn *libcFunc) uintptr { +func sysvicall0(fn libcFunc) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 0 - // TODO(rsc): Why is noescape necessary here and below? - libcall.args = uintptr(noescape(unsafe.Pointer(&fn))) // it's unused but must be non-nil, otherwise crashes + libcall.args = uintptr(fn) // it's unused but must be non-nil, otherwise crashes asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) return libcall.r1 } //go:nosplit -func sysvicall1(fn *libcFunc, a1 uintptr) uintptr { +func sysvicall1(fn libcFunc, a1 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 1 + // TODO(rsc): Why is noescape necessary here and below? libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) return libcall.r1 } //go:nosplit -func sysvicall2(fn *libcFunc, a1, a2 uintptr) uintptr { +func sysvicall2(fn libcFunc, a1, a2 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 2 libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) @@ -60,9 +42,9 @@ func sysvicall2(fn *libcFunc, a1, a2 uintptr) uintptr { } //go:nosplit -func sysvicall3(fn *libcFunc, a1, a2, a3 uintptr) uintptr { +func sysvicall3(fn libcFunc, a1, a2, a3 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 3 libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) @@ -70,9 +52,9 @@ func sysvicall3(fn *libcFunc, a1, a2, a3 uintptr) uintptr { } //go:nosplit -func sysvicall4(fn *libcFunc, a1, a2, a3, a4 uintptr) uintptr { +func sysvicall4(fn libcFunc, a1, a2, a3, a4 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 4 libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) @@ -80,9 +62,9 @@ func sysvicall4(fn *libcFunc, a1, a2, a3, a4 uintptr) uintptr { } //go:nosplit -func sysvicall5(fn *libcFunc, a1, a2, a3, a4, a5 uintptr) uintptr { +func sysvicall5(fn libcFunc, a1, a2, a3, a4, a5 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 5 libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) @@ -90,9 +72,9 @@ func sysvicall5(fn *libcFunc, a1, a2, a3, a4, a5 uintptr) uintptr { } //go:nosplit -func sysvicall6(fn *libcFunc, a1, a2, a3, a4, a5, a6 uintptr) uintptr { +func sysvicall6(fn libcFunc, a1, a2, a3, a4, a5, a6 uintptr) uintptr { libcall := &getg().m.libcall - libcall.fn = uintptr(unsafe.Pointer(fn)) + libcall.fn = uintptr(fn) libcall.n = 6 libcall.args = uintptr(noescape(unsafe.Pointer(&a1))) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(libcall)) diff --git a/src/runtime/os_solaris.h b/src/runtime/os_solaris.h deleted file mode 100644 index 3d9e1a240..000000000 --- a/src/runtime/os_solaris.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - - -typedef uintptr kevent_udata; - -struct sigaction; - -void runtime·sigpanic(void); - -void runtime·setitimer(int32, Itimerval*, Itimerval*); -void runtime·sigaction(int32, struct SigactionT*, struct SigactionT*); -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -void runtime·sigprocmask(int32, Sigset*, Sigset*); -void runtime·unblocksignals(void); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - - -void runtime·raisesigpipe(void); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); -void runtime·sigpanic(void); - -enum { - SS_DISABLE = 2, - SIG_BLOCK = 1, - SIG_UNBLOCK = 2, - SIG_SETMASK = 3, - NSIG = 73, /* number of signals in runtime·SigTab array */ - SI_USER = 0, - _UC_SIGMASK = 0x01, - _UC_CPU = 0x04, - RLIMIT_AS = 10, -}; - -typedef struct Rlimit Rlimit; -struct Rlimit { - int64 rlim_cur; - int64 rlim_max; -}; -int32 runtime·getrlimit(int32, Rlimit*); - -// Call an external library function described by {fn, a0, ..., an}, with -// SysV conventions, switching to os stack during the call, if necessary. -uintptr runtime·sysvicall0(uintptr fn); -uintptr runtime·sysvicall1(uintptr fn, uintptr a1); -uintptr runtime·sysvicall2(uintptr fn, uintptr a1, uintptr a2); -uintptr runtime·sysvicall3(uintptr fn, uintptr a1, uintptr a2, uintptr a3); -uintptr runtime·sysvicall4(uintptr fn, uintptr a1, uintptr a2, uintptr a3, uintptr a4); -uintptr runtime·sysvicall5(uintptr fn, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5); -uintptr runtime·sysvicall6(uintptr fn, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5, uintptr a6); -void runtime·asmsysvicall6(void *c); - -void runtime·miniterrno(void *fn); diff --git a/src/runtime/os_windows.c b/src/runtime/os_windows.c deleted file mode 100644 index b8b8eda5f..000000000 --- a/src/runtime/os_windows.c +++ /dev/null @@ -1,636 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "type.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -#pragma dynimport runtime·AddVectoredExceptionHandler AddVectoredExceptionHandler "kernel32.dll" -#pragma dynimport runtime·CloseHandle CloseHandle "kernel32.dll" -#pragma dynimport runtime·CreateEvent CreateEventA "kernel32.dll" -#pragma dynimport runtime·CreateThread CreateThread "kernel32.dll" -#pragma dynimport runtime·CreateWaitableTimer CreateWaitableTimerA "kernel32.dll" -#pragma dynimport runtime·CryptAcquireContextW CryptAcquireContextW "advapi32.dll" -#pragma dynimport runtime·CryptGenRandom CryptGenRandom "advapi32.dll" -#pragma dynimport runtime·CryptReleaseContext CryptReleaseContext "advapi32.dll" -#pragma dynimport runtime·DuplicateHandle DuplicateHandle "kernel32.dll" -#pragma dynimport runtime·ExitProcess ExitProcess "kernel32.dll" -#pragma dynimport runtime·FreeEnvironmentStringsW FreeEnvironmentStringsW "kernel32.dll" -#pragma dynimport runtime·GetEnvironmentStringsW GetEnvironmentStringsW "kernel32.dll" -#pragma dynimport runtime·GetProcAddress GetProcAddress "kernel32.dll" -#pragma dynimport runtime·GetStdHandle GetStdHandle "kernel32.dll" -#pragma dynimport runtime·GetSystemInfo GetSystemInfo "kernel32.dll" -#pragma dynimport runtime·GetThreadContext GetThreadContext "kernel32.dll" -#pragma dynimport runtime·LoadLibrary LoadLibraryW "kernel32.dll" -#pragma dynimport runtime·LoadLibraryA LoadLibraryA "kernel32.dll" -#pragma dynimport runtime·NtWaitForSingleObject NtWaitForSingleObject "ntdll.dll" -#pragma dynimport runtime·ResumeThread ResumeThread "kernel32.dll" -#pragma dynimport runtime·SetConsoleCtrlHandler SetConsoleCtrlHandler "kernel32.dll" -#pragma dynimport runtime·SetEvent SetEvent "kernel32.dll" -#pragma dynimport runtime·SetProcessPriorityBoost SetProcessPriorityBoost "kernel32.dll" -#pragma dynimport runtime·SetThreadPriority SetThreadPriority "kernel32.dll" -#pragma dynimport runtime·SetUnhandledExceptionFilter SetUnhandledExceptionFilter "kernel32.dll" -#pragma dynimport runtime·SetWaitableTimer SetWaitableTimer "kernel32.dll" -#pragma dynimport runtime·Sleep Sleep "kernel32.dll" -#pragma dynimport runtime·SuspendThread SuspendThread "kernel32.dll" -#pragma dynimport runtime·WaitForSingleObject WaitForSingleObject "kernel32.dll" -#pragma dynimport runtime·WriteFile WriteFile "kernel32.dll" -#pragma dynimport runtime·timeBeginPeriod timeBeginPeriod "winmm.dll" - -extern void *runtime·AddVectoredExceptionHandler; -extern void *runtime·CloseHandle; -extern void *runtime·CreateEvent; -extern void *runtime·CreateThread; -extern void *runtime·CreateWaitableTimer; -extern void *runtime·CryptAcquireContextW; -extern void *runtime·CryptGenRandom; -extern void *runtime·CryptReleaseContext; -extern void *runtime·DuplicateHandle; -extern void *runtime·ExitProcess; -extern void *runtime·FreeEnvironmentStringsW; -extern void *runtime·GetEnvironmentStringsW; -extern void *runtime·GetProcAddress; -extern void *runtime·GetStdHandle; -extern void *runtime·GetSystemInfo; -extern void *runtime·GetThreadContext; -extern void *runtime·LoadLibrary; -extern void *runtime·LoadLibraryA; -extern void *runtime·NtWaitForSingleObject; -extern void *runtime·ResumeThread; -extern void *runtime·SetConsoleCtrlHandler; -extern void *runtime·SetEvent; -extern void *runtime·SetProcessPriorityBoost; -extern void *runtime·SetThreadPriority; -extern void *runtime·SetUnhandledExceptionFilter; -extern void *runtime·SetWaitableTimer; -extern void *runtime·Sleep; -extern void *runtime·SuspendThread; -extern void *runtime·WaitForSingleObject; -extern void *runtime·WriteFile; -extern void *runtime·timeBeginPeriod; - -#pragma dataflag NOPTR -void *runtime·GetQueuedCompletionStatusEx; - -extern uintptr runtime·externalthreadhandlerp; -void runtime·externalthreadhandler(void); -void runtime·exceptiontramp(void); -void runtime·firstcontinuetramp(void); -void runtime·lastcontinuetramp(void); - -#pragma textflag NOSPLIT -uintptr -runtime·getLoadLibrary(void) -{ - return (uintptr)runtime·LoadLibrary; -} - -#pragma textflag NOSPLIT -uintptr -runtime·getGetProcAddress(void) -{ - return (uintptr)runtime·GetProcAddress; -} - -static int32 -getproccount(void) -{ - SystemInfo info; - - runtime·stdcall1(runtime·GetSystemInfo, (uintptr)&info); - return info.dwNumberOfProcessors; -} - -void -runtime·osinit(void) -{ - void *kernel32; - void *addVectoredContinueHandler; - - kernel32 = runtime·stdcall1(runtime·LoadLibraryA, (uintptr)"kernel32.dll"); - - runtime·externalthreadhandlerp = (uintptr)runtime·externalthreadhandler; - - runtime·stdcall2(runtime·AddVectoredExceptionHandler, 1, (uintptr)runtime·exceptiontramp); - addVectoredContinueHandler = nil; - if(kernel32 != nil) - addVectoredContinueHandler = runtime·stdcall2(runtime·GetProcAddress, (uintptr)kernel32, (uintptr)"AddVectoredContinueHandler"); - if(addVectoredContinueHandler == nil || sizeof(void*) == 4) { - // use SetUnhandledExceptionFilter for windows-386 or - // if VectoredContinueHandler is unavailable. - // note: SetUnhandledExceptionFilter handler won't be called, if debugging. - runtime·stdcall1(runtime·SetUnhandledExceptionFilter, (uintptr)runtime·lastcontinuetramp); - } else { - runtime·stdcall2(addVectoredContinueHandler, 1, (uintptr)runtime·firstcontinuetramp); - runtime·stdcall2(addVectoredContinueHandler, 0, (uintptr)runtime·lastcontinuetramp); - } - - runtime·stdcall2(runtime·SetConsoleCtrlHandler, (uintptr)runtime·ctrlhandler, 1); - - runtime·stdcall1(runtime·timeBeginPeriod, 1); - - runtime·ncpu = getproccount(); - - // Windows dynamic priority boosting assumes that a process has different types - // of dedicated threads -- GUI, IO, computational, etc. Go processes use - // equivalent threads that all do a mix of GUI, IO, computations, etc. - // In such context dynamic priority boosting does nothing but harm, so we turn it off. - runtime·stdcall2(runtime·SetProcessPriorityBoost, -1, 1); - - if(kernel32 != nil) { - runtime·GetQueuedCompletionStatusEx = runtime·stdcall2(runtime·GetProcAddress, (uintptr)kernel32, (uintptr)"GetQueuedCompletionStatusEx"); - } -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - uintptr handle; - *rnd = nil; - *rnd_len = 0; - if(runtime·stdcall5(runtime·CryptAcquireContextW, (uintptr)&handle, (uintptr)nil, (uintptr)nil, - 1 /* PROV_RSA_FULL */, - 0xf0000000U /* CRYPT_VERIFYCONTEXT */) != 0) { - static byte random_data[HashRandomBytes]; - if(runtime·stdcall3(runtime·CryptGenRandom, handle, HashRandomBytes, (uintptr)&random_data[0])) { - *rnd = random_data; - *rnd_len = HashRandomBytes; - } - runtime·stdcall2(runtime·CryptReleaseContext, handle, 0); - } -} - -void -runtime·goenvs(void) -{ - extern Slice runtime·envs; - - uint16 *env; - String *s; - int32 i, n; - uint16 *p; - - env = runtime·stdcall0(runtime·GetEnvironmentStringsW); - - n = 0; - for(p=env; *p; n++) - p += runtime·findnullw(p)+1; - - runtime·envs = runtime·makeStringSlice(n); - s = (String*)runtime·envs.array; - - p = env; - for(i=0; i<n; i++) { - s[i] = runtime·gostringw(p); - p += runtime·findnullw(p)+1; - } - - runtime·stdcall1(runtime·FreeEnvironmentStringsW, (uintptr)env); -} - -#pragma textflag NOSPLIT -void -runtime·exit(int32 code) -{ - runtime·stdcall1(runtime·ExitProcess, code); -} - -#pragma textflag NOSPLIT -int32 -runtime·write(uintptr fd, void *buf, int32 n) -{ - void *handle; - uint32 written; - - written = 0; - switch(fd) { - case 1: - handle = runtime·stdcall1(runtime·GetStdHandle, -11); - break; - case 2: - handle = runtime·stdcall1(runtime·GetStdHandle, -12); - break; - default: - // assume fd is real windows handle. - handle = (void*)fd; - break; - } - runtime·stdcall5(runtime·WriteFile, (uintptr)handle, (uintptr)buf, n, (uintptr)&written, 0); - return written; -} - -#define INFINITE ((uintptr)0xFFFFFFFF) - -#pragma textflag NOSPLIT -int32 -runtime·semasleep(int64 ns) -{ - // store ms in ns to save stack space - if(ns < 0) - ns = INFINITE; - else { - ns = runtime·timediv(ns, 1000000, nil); - if(ns == 0) - ns = 1; - } - if(runtime·stdcall2(runtime·WaitForSingleObject, (uintptr)g->m->waitsema, ns) != 0) - return -1; // timeout - return 0; -} - -#pragma textflag NOSPLIT -void -runtime·semawakeup(M *mp) -{ - runtime·stdcall1(runtime·SetEvent, mp->waitsema); -} - -#pragma textflag NOSPLIT -uintptr -runtime·semacreate(void) -{ - return (uintptr)runtime·stdcall4(runtime·CreateEvent, 0, 0, 0, 0); -} - -#define STACK_SIZE_PARAM_IS_A_RESERVATION ((uintptr)0x00010000) - -void -runtime·newosproc(M *mp, void *stk) -{ - void *thandle; - - USED(stk); - - thandle = runtime·stdcall6(runtime·CreateThread, - (uintptr)nil, 0x20000, (uintptr)runtime·tstart_stdcall, (uintptr)mp, - STACK_SIZE_PARAM_IS_A_RESERVATION, (uintptr)nil); - if(thandle == nil) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), runtime·getlasterror()); - runtime·throw("runtime.newosproc"); - } -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - USED(mp); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - uintptr thandle; - - // -1 = current process, -2 = current thread - runtime·stdcall7(runtime·DuplicateHandle, -1, -2, -1, (uintptr)&thandle, 0, 0, DUPLICATE_SAME_ACCESS); - runtime·atomicstoreuintptr(&g->m->thread, thandle); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·stdcall1(runtime·CloseHandle, g->m->thread); - g->m->thread = 0; -} - -// Described in http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/ -typedef struct KSYSTEM_TIME { - uint32 LowPart; - int32 High1Time; - int32 High2Time; -} KSYSTEM_TIME; - -#pragma dataflag NOPTR -const KSYSTEM_TIME* INTERRUPT_TIME = (KSYSTEM_TIME*)0x7ffe0008; -#pragma dataflag NOPTR -const KSYSTEM_TIME* SYSTEM_TIME = (KSYSTEM_TIME*)0x7ffe0014; - -static void badsystime(void); - -#pragma textflag NOSPLIT -int64 -runtime·systime(KSYSTEM_TIME *timeaddr) -{ - KSYSTEM_TIME t; - int32 i; - void (*fn)(void); - - for(i = 1; i < 10000; i++) { - // these fields must be read in that order (see URL above) - t.High1Time = timeaddr->High1Time; - t.LowPart = timeaddr->LowPart; - t.High2Time = timeaddr->High2Time; - if(t.High1Time == t.High2Time) - return (int64)t.High1Time<<32 | t.LowPart; - if((i%100) == 0) - runtime·osyield(); - } - fn = badsystime; - runtime·onM(&fn); - return 0; -} - -#pragma textflag NOSPLIT -int64 -runtime·unixnano(void) -{ - return (runtime·systime(SYSTEM_TIME) - 116444736000000000LL) * 100LL; -} - -static void -badsystime(void) -{ - runtime·throw("interrupt/system time is changing too fast"); -} - -#pragma textflag NOSPLIT -int64 -runtime·nanotime(void) -{ - return runtime·systime(INTERRUPT_TIME) * 100LL; -} - -// Calling stdcall on os stack. -#pragma textflag NOSPLIT -static void* -stdcall(void *fn) -{ - g->m->libcall.fn = (uintptr)fn; - if(g->m->profilehz != 0) { - // leave pc/sp for cpu profiler - g->m->libcallg = g; - g->m->libcallpc = (uintptr)runtime·getcallerpc(&fn); - // sp must be the last, because once async cpu profiler finds - // all three values to be non-zero, it will use them - g->m->libcallsp = (uintptr)runtime·getcallersp(&fn); - } - runtime·asmcgocall(runtime·asmstdcall, &g->m->libcall); - g->m->libcallsp = 0; - return (void*)g->m->libcall.r1; -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall0(void *fn) -{ - g->m->libcall.n = 0; - g->m->libcall.args = (uintptr)&fn; // it's unused but must be non-nil, otherwise crashes - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall1(void *fn, uintptr a0) -{ - USED(a0); - g->m->libcall.n = 1; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall2(void *fn, uintptr a0, uintptr a1) -{ - USED(a0, a1); - g->m->libcall.n = 2; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall3(void *fn, uintptr a0, uintptr a1, uintptr a2) -{ - USED(a0, a1, a2); - g->m->libcall.n = 3; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall4(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3) -{ - USED(a0, a1, a2, a3); - g->m->libcall.n = 4; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall5(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4) -{ - USED(a0, a1, a2, a3, a4); - g->m->libcall.n = 5; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall6(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5) -{ - USED(a0, a1, a2, a3, a4, a5); - g->m->libcall.n = 6; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -#pragma textflag NOSPLIT -void* -runtime·stdcall7(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5, uintptr a6) -{ - USED(a0, a1, a2, a3, a4, a5, a6); - g->m->libcall.n = 7; - g->m->libcall.args = (uintptr)&a0; - return stdcall(fn); -} - -extern void runtime·usleep1(uint32); - -#pragma textflag NOSPLIT -void -runtime·osyield(void) -{ - runtime·usleep1(1); -} - -#pragma textflag NOSPLIT -void -runtime·usleep(uint32 us) -{ - // Have 1us units; want 100ns units. - runtime·usleep1(10*us); -} - -uint32 -runtime·issigpanic(uint32 code) -{ - switch(code) { - case EXCEPTION_ACCESS_VIOLATION: - case EXCEPTION_INT_DIVIDE_BY_ZERO: - case EXCEPTION_INT_OVERFLOW: - case EXCEPTION_FLT_DENORMAL_OPERAND: - case EXCEPTION_FLT_DIVIDE_BY_ZERO: - case EXCEPTION_FLT_INEXACT_RESULT: - case EXCEPTION_FLT_OVERFLOW: - case EXCEPTION_FLT_UNDERFLOW: - case EXCEPTION_BREAKPOINT: - return 1; - } - return 0; -} - -void -runtime·initsig(void) -{ - // following line keeps these functions alive at link stage - // if there's a better way please write it here - void *e = runtime·exceptiontramp; - void *f = runtime·firstcontinuetramp; - void *l = runtime·lastcontinuetramp; - USED(e); - USED(f); - USED(l); -} - -uint32 -runtime·ctrlhandler1(uint32 type) -{ - int32 s; - - switch(type) { - case CTRL_C_EVENT: - case CTRL_BREAK_EVENT: - s = SIGINT; - break; - default: - return 0; - } - - if(runtime·sigsend(s)) - return 1; - runtime·exit(2); // SIGINT, SIGTERM, etc - return 0; -} - -extern void runtime·dosigprof(Context *r, G *gp, M *mp); -extern void runtime·profileloop(void); -#pragma dataflag NOPTR -static void *profiletimer; - -static void -profilem(M *mp) -{ - extern M runtime·m0; - extern uint32 runtime·tls0[]; - byte rbuf[sizeof(Context)+15]; - Context *r; - void *tls; - G *gp; - - tls = mp->tls; - if(mp == &runtime·m0) - tls = runtime·tls0; - gp = *(G**)tls; - - // align Context to 16 bytes - r = (Context*)((uintptr)(&rbuf[15]) & ~15); - r->ContextFlags = CONTEXT_CONTROL; - runtime·stdcall2(runtime·GetThreadContext, (uintptr)mp->thread, (uintptr)r); - runtime·dosigprof(r, gp, mp); -} - -void -runtime·profileloop1(void) -{ - M *mp, *allm; - uintptr thread; - - runtime·stdcall2(runtime·SetThreadPriority, -2, THREAD_PRIORITY_HIGHEST); - - for(;;) { - runtime·stdcall2(runtime·WaitForSingleObject, (uintptr)profiletimer, -1); - allm = runtime·atomicloadp(&runtime·allm); - for(mp = allm; mp != nil; mp = mp->alllink) { - thread = runtime·atomicloaduintptr(&mp->thread); - // Do not profile threads blocked on Notes, - // this includes idle worker threads, - // idle timer thread, idle heap scavenger, etc. - if(thread == 0 || mp->profilehz == 0 || mp->blocked) - continue; - runtime·stdcall1(runtime·SuspendThread, (uintptr)thread); - if(mp->profilehz != 0 && !mp->blocked) - profilem(mp); - runtime·stdcall1(runtime·ResumeThread, (uintptr)thread); - } - } -} - -void -runtime·resetcpuprofiler(int32 hz) -{ - static Mutex lock; - void *timer, *thread; - int32 ms; - int64 due; - - runtime·lock(&lock); - if(profiletimer == nil) { - timer = runtime·stdcall3(runtime·CreateWaitableTimer, (uintptr)nil, (uintptr)nil, (uintptr)nil); - runtime·atomicstorep(&profiletimer, timer); - thread = runtime·stdcall6(runtime·CreateThread, - (uintptr)nil, (uintptr)nil, (uintptr)runtime·profileloop, (uintptr)nil, (uintptr)nil, (uintptr)nil); - runtime·stdcall2(runtime·SetThreadPriority, (uintptr)thread, THREAD_PRIORITY_HIGHEST); - runtime·stdcall1(runtime·CloseHandle, (uintptr)thread); - } - runtime·unlock(&lock); - - ms = 0; - due = 1LL<<63; - if(hz > 0) { - ms = 1000 / hz; - if(ms == 0) - ms = 1; - due = ms * -10000; - } - runtime·stdcall6(runtime·SetWaitableTimer, - (uintptr)profiletimer, (uintptr)&due, ms, (uintptr)nil, (uintptr)nil, (uintptr)nil); - runtime·atomicstore((uint32*)&g->m->profilehz, hz); -} - -uintptr -runtime·memlimit(void) -{ - return 0; -} - -#pragma dataflag NOPTR -int8 runtime·badsignalmsg[] = "runtime: signal received on thread not created by Go.\n"; -int32 runtime·badsignallen = sizeof runtime·badsignalmsg - 1; - -void -runtime·crash(void) -{ - // TODO: This routine should do whatever is needed - // to make the Windows program abort/crash as it - // would if Go was not intercepting signals. - // On Unix the routine would remove the custom signal - // handler and then raise a signal (like SIGABRT). - // Something like that should happen here. - // It's okay to leave this empty for now: if crash returns - // the ordinary exit-after-panic happens. -} diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 1528d2fd1..097b5d629 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -4,28 +4,8 @@ package runtime -import "unsafe" - type stdFunction *byte -func stdcall0(fn stdFunction) uintptr -func stdcall1(fn stdFunction, a0 uintptr) uintptr -func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr -func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr -func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr -func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr -func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr -func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr - -func asmstdcall(fn unsafe.Pointer) -func getlasterror() uint32 -func setlasterror(err uint32) -func usleep1(usec uint32) -func netpollinit() -func netpollopen(fd uintptr, pd *pollDesc) int32 -func netpollclose(fd uintptr) int32 -func netpollarm(pd *pollDesc, mode int) - func os_sigpipe() { gothrow("too many writes on closed pipe") } diff --git a/src/runtime/os_windows.h b/src/runtime/os_windows.h deleted file mode 100644 index d5d168d77..000000000 --- a/src/runtime/os_windows.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -extern void *runtime·LoadLibrary; -extern void *runtime·GetProcAddress; -extern void *runtime·GetQueuedCompletionStatusEx; - -// Call a Windows function with stdcall conventions, -// and switch to os stack during the call. -void runtime·asmstdcall(void *c); -void *runtime·stdcall0(void *fn); -void *runtime·stdcall1(void *fn, uintptr a0); -void *runtime·stdcall2(void *fn, uintptr a0, uintptr a1); -void *runtime·stdcall3(void *fn, uintptr a0, uintptr a1, uintptr a2); -void *runtime·stdcall4(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3); -void *runtime·stdcall5(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4); -void *runtime·stdcall6(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5); -void *runtime·stdcall7(void *fn, uintptr a0, uintptr a1, uintptr a2, uintptr a3, uintptr a4, uintptr a5, uintptr a6); - -uint32 runtime·getlasterror(void); -void runtime·setlasterror(uint32 err); - -// Function to be called by windows CreateThread -// to start new os thread. -uint32 runtime·tstart_stdcall(M *newm); - -uint32 runtime·issigpanic(uint32); -void runtime·sigpanic(void); -uint32 runtime·ctrlhandler(uint32 type); - -// Windows dll function to go callback entry. -byte *runtime·compilecallback(Eface fn, bool cleanstack); -void *runtime·callbackasm(void); - -void runtime·install_exception_handler(void); -void runtime·remove_exception_handler(void); - -// TODO(brainman): should not need those -enum { - NSIG = 65, -}; diff --git a/src/runtime/os_windows_386.c b/src/runtime/os_windows_386.c deleted file mode 100644 index 9962f0dc2..000000000 --- a/src/runtime/os_windows_386.c +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" - -void -runtime·dumpregs(Context *r) -{ - runtime·printf("eax %x\n", r->Eax); - runtime·printf("ebx %x\n", r->Ebx); - runtime·printf("ecx %x\n", r->Ecx); - runtime·printf("edx %x\n", r->Edx); - runtime·printf("edi %x\n", r->Edi); - runtime·printf("esi %x\n", r->Esi); - runtime·printf("ebp %x\n", r->Ebp); - runtime·printf("esp %x\n", r->Esp); - runtime·printf("eip %x\n", r->Eip); - runtime·printf("eflags %x\n", r->EFlags); - runtime·printf("cs %x\n", r->SegCs); - runtime·printf("fs %x\n", r->SegFs); - runtime·printf("gs %x\n", r->SegGs); -} - -bool -runtime·isgoexception(ExceptionRecord *info, Context *r) -{ - extern byte runtime·text[], runtime·etext[]; - - // Only handle exception if executing instructions in Go binary - // (not Windows library code). - if(r->Eip < (uint32)runtime·text || (uint32)runtime·etext < r->Eip) - return false; - - if(!runtime·issigpanic(info->ExceptionCode)) - return false; - - return true; -} - -// Called by sigtramp from Windows VEH handler. -// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION) -// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH). -uint32 -runtime·exceptionhandler(ExceptionRecord *info, Context *r, G *gp) -{ - uintptr *sp; - - if(!runtime·isgoexception(info, r)) - return EXCEPTION_CONTINUE_SEARCH; - - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = info->ExceptionCode; - gp->sigcode0 = info->ExceptionInformation[0]; - gp->sigcode1 = info->ExceptionInformation[1]; - gp->sigpc = r->Eip; - - // Only push runtime·sigpanic if r->eip != 0. - // If r->eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->Eip != 0) { - sp = (uintptr*)r->Esp; - *--sp = r->Eip; - r->Esp = (uintptr)sp; - } - r->Eip = (uintptr)runtime·sigpanic; - return EXCEPTION_CONTINUE_EXECUTION; -} - -// lastcontinuehandler is reached, because runtime cannot handle -// current exception. lastcontinuehandler will print crash info and exit. -uint32 -runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) -{ - bool crash; - - if(runtime·panicking) // traceback already printed - runtime·exit(2); - runtime·panicking = 1; - - runtime·printf("Exception %x %p %p %p\n", info->ExceptionCode, - (uintptr)info->ExceptionInformation[0], (uintptr)info->ExceptionInformation[1], (uintptr)r->Eip); - - runtime·printf("PC=%x\n", r->Eip); - if(g->m->lockedg != nil && g->m->ncgo > 0 && gp == g->m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = g->m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback(&crash)){ - runtime·tracebacktrap(r->Eip, r->Esp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); - return 0; // not reached -} - -void -runtime·sigenable(uint32 sig) -{ - USED(sig); -} - -void -runtime·sigdisable(uint32 sig) -{ - USED(sig); -} - -void -runtime·dosigprof(Context *r, G *gp, M *mp) -{ - runtime·sigprof((uint8*)r->Eip, (uint8*)r->Esp, nil, gp, mp); -} diff --git a/src/runtime/os_windows_amd64.c b/src/runtime/os_windows_amd64.c deleted file mode 100644 index e4617e4ce..000000000 --- a/src/runtime/os_windows_amd64.c +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" - -void -runtime·dumpregs(Context *r) -{ - runtime·printf("rax %X\n", r->Rax); - runtime·printf("rbx %X\n", r->Rbx); - runtime·printf("rcx %X\n", r->Rcx); - runtime·printf("rdx %X\n", r->Rdx); - runtime·printf("rdi %X\n", r->Rdi); - runtime·printf("rsi %X\n", r->Rsi); - runtime·printf("rbp %X\n", r->Rbp); - runtime·printf("rsp %X\n", r->Rsp); - runtime·printf("r8 %X\n", r->R8 ); - runtime·printf("r9 %X\n", r->R9 ); - runtime·printf("r10 %X\n", r->R10); - runtime·printf("r11 %X\n", r->R11); - runtime·printf("r12 %X\n", r->R12); - runtime·printf("r13 %X\n", r->R13); - runtime·printf("r14 %X\n", r->R14); - runtime·printf("r15 %X\n", r->R15); - runtime·printf("rip %X\n", r->Rip); - runtime·printf("rflags %X\n", r->EFlags); - runtime·printf("cs %X\n", (uint64)r->SegCs); - runtime·printf("fs %X\n", (uint64)r->SegFs); - runtime·printf("gs %X\n", (uint64)r->SegGs); -} - -bool -runtime·isgoexception(ExceptionRecord *info, Context *r) -{ - extern byte runtime·text[], runtime·etext[]; - - // Only handle exception if executing instructions in Go binary - // (not Windows library code). - if(r->Rip < (uint64)runtime·text || (uint64)runtime·etext < r->Rip) - return false; - - if(!runtime·issigpanic(info->ExceptionCode)) - return false; - - return true; -} - -// Called by sigtramp from Windows VEH handler. -// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION) -// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH). -uint32 -runtime·exceptionhandler(ExceptionRecord *info, Context *r, G *gp) -{ - uintptr *sp; - - if(!runtime·isgoexception(info, r)) - return EXCEPTION_CONTINUE_SEARCH; - - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = info->ExceptionCode; - gp->sigcode0 = info->ExceptionInformation[0]; - gp->sigcode1 = info->ExceptionInformation[1]; - gp->sigpc = r->Rip; - - // Only push runtime·sigpanic if r->rip != 0. - // If r->rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->Rip != 0) { - sp = (uintptr*)r->Rsp; - *--sp = r->Rip; - r->Rsp = (uintptr)sp; - } - r->Rip = (uintptr)runtime·sigpanic; - return EXCEPTION_CONTINUE_EXECUTION; -} - -// It seems Windows searches ContinueHandler's list even -// if ExceptionHandler returns EXCEPTION_CONTINUE_EXECUTION. -// firstcontinuehandler will stop that search, -// if exceptionhandler did the same earlier. -uint32 -runtime·firstcontinuehandler(ExceptionRecord *info, Context *r, G *gp) -{ - USED(gp); - if(!runtime·isgoexception(info, r)) - return EXCEPTION_CONTINUE_SEARCH; - return EXCEPTION_CONTINUE_EXECUTION; -} - -// lastcontinuehandler is reached, because runtime cannot handle -// current exception. lastcontinuehandler will print crash info and exit. -uint32 -runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) -{ - bool crash; - - if(runtime·panicking) // traceback already printed - runtime·exit(2); - runtime·panicking = 1; - - runtime·printf("Exception %x %p %p %p\n", info->ExceptionCode, - info->ExceptionInformation[0], info->ExceptionInformation[1], r->Rip); - - - runtime·printf("PC=%X\n", r->Rip); - if(g->m->lockedg != nil && g->m->ncgo > 0 && gp == g->m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = g->m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback(&crash)){ - runtime·tracebacktrap(r->Rip, r->Rsp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); - return 0; // not reached -} - -void -runtime·sigenable(uint32 sig) -{ - USED(sig); -} - -void -runtime·sigdisable(uint32 sig) -{ - USED(sig); -} - -void -runtime·dosigprof(Context *r, G *gp, M *mp) -{ - runtime·sigprof((uint8*)r->Rip, (uint8*)r->Rsp, nil, gp, mp); -} diff --git a/src/runtime/panic.c b/src/runtime/panic.c deleted file mode 100644 index 24eb6dbfe..000000000 --- a/src/runtime/panic.c +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "stack.h" -#include "malloc.h" -#include "textflag.h" - -// Code related to defer, panic and recover. - -// TODO: remove once code is moved to Go -extern Defer* runtime·newdefer(int32 siz); -extern runtime·freedefer(Defer *d); - -uint32 runtime·panicking; -static Mutex paniclk; - -void -runtime·deferproc_m(void) -{ - int32 siz; - FuncVal *fn; - uintptr argp; - uintptr callerpc; - Defer *d; - - siz = g->m->scalararg[0]; - fn = g->m->ptrarg[0]; - argp = g->m->scalararg[1]; - callerpc = g->m->scalararg[2]; - g->m->ptrarg[0] = nil; - g->m->scalararg[1] = 0; - - d = runtime·newdefer(siz); - if(d->panic != nil) - runtime·throw("deferproc: d->panic != nil after newdefer"); - d->fn = fn; - d->pc = callerpc; - d->argp = argp; - runtime·memmove(d+1, (void*)argp, siz); -} - -// Unwind the stack after a deferred function calls recover -// after a panic. Then arrange to continue running as though -// the caller of the deferred function returned normally. -void -runtime·recovery_m(G *gp) -{ - void *argp; - uintptr pc; - - // Info about defer passed in G struct. - argp = (void*)gp->sigcode0; - pc = (uintptr)gp->sigcode1; - - // d's arguments need to be in the stack. - if(argp != nil && ((uintptr)argp < gp->stack.lo || gp->stack.hi < (uintptr)argp)) { - runtime·printf("recover: %p not in [%p, %p]\n", argp, gp->stack.lo, gp->stack.hi); - runtime·throw("bad recovery"); - } - - // Make the deferproc for this d return again, - // this time returning 1. The calling function will - // jump to the standard return epilogue. - // The -2*sizeof(uintptr) makes up for the - // two extra words that are on the stack at - // each call to deferproc. - // (The pc we're returning to does pop pop - // before it tests the return value.) - // On the arm there are 2 saved LRs mixed in too. - if(thechar == '5') - gp->sched.sp = (uintptr)argp - 4*sizeof(uintptr); - else - gp->sched.sp = (uintptr)argp - 2*sizeof(uintptr); - gp->sched.pc = pc; - gp->sched.lr = 0; - gp->sched.ret = 1; - runtime·gogo(&gp->sched); -} - -void -runtime·startpanic_m(void) -{ - if(runtime·mheap.cachealloc.size == 0) { // very early - runtime·printf("runtime: panic before malloc heap initialized\n"); - g->m->mallocing = 1; // tell rest of panic not to try to malloc - } else if(g->m->mcache == nil) // can happen if called from signal handler or throw - g->m->mcache = runtime·allocmcache(); - switch(g->m->dying) { - case 0: - g->m->dying = 1; - if(g != nil) { - g->writebuf.array = nil; - g->writebuf.len = 0; - g->writebuf.cap = 0; - } - runtime·xadd(&runtime·panicking, 1); - runtime·lock(&paniclk); - if(runtime·debug.schedtrace > 0 || runtime·debug.scheddetail > 0) - runtime·schedtrace(true); - runtime·freezetheworld(); - return; - case 1: - // Something failed while panicing, probably the print of the - // argument to panic(). Just print a stack trace and exit. - g->m->dying = 2; - runtime·printf("panic during panic\n"); - runtime·dopanic(0); - runtime·exit(3); - case 2: - // This is a genuine bug in the runtime, we couldn't even - // print the stack trace successfully. - g->m->dying = 3; - runtime·printf("stack trace unavailable\n"); - runtime·exit(4); - default: - // Can't even print! Just exit. - runtime·exit(5); - } -} - -void -runtime·dopanic_m(void) -{ - G *gp; - uintptr sp, pc; - static bool didothers; - bool crash; - int32 t; - - gp = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - pc = g->m->scalararg[0]; - sp = g->m->scalararg[1]; - g->m->scalararg[1] = 0; - if(gp->sig != 0) - runtime·printf("[signal %x code=%p addr=%p pc=%p]\n", - gp->sig, gp->sigcode0, gp->sigcode1, gp->sigpc); - - if((t = runtime·gotraceback(&crash)) > 0){ - if(gp != gp->m->g0) { - runtime·printf("\n"); - runtime·goroutineheader(gp); - runtime·traceback(pc, sp, 0, gp); - } else if(t >= 2 || g->m->throwing > 0) { - runtime·printf("\nruntime stack:\n"); - runtime·traceback(pc, sp, 0, gp); - } - if(!didothers) { - didothers = true; - runtime·tracebackothers(gp); - } - } - runtime·unlock(&paniclk); - if(runtime·xadd(&runtime·panicking, -1) != 0) { - // Some other m is panicking too. - // Let it print what it needs to print. - // Wait forever without chewing up cpu. - // It will exit when it's done. - static Mutex deadlock; - runtime·lock(&deadlock); - runtime·lock(&deadlock); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); -} - -#pragma textflag NOSPLIT -bool -runtime·canpanic(G *gp) -{ - M *m; - uint32 status; - - // Note that g is m->gsignal, different from gp. - // Note also that g->m can change at preemption, so m can go stale - // if this function ever makes a function call. - m = g->m; - - // Is it okay for gp to panic instead of crashing the program? - // Yes, as long as it is running Go code, not runtime code, - // and not stuck in a system call. - if(gp == nil || gp != m->curg) - return false; - if(m->locks-m->softfloat != 0 || m->mallocing != 0 || m->throwing != 0 || m->gcing != 0 || m->dying != 0) - return false; - status = runtime·readgstatus(gp); - if((status&~Gscan) != Grunning || gp->syscallsp != 0) - return false; -#ifdef GOOS_windows - if(m->libcallsp != 0) - return false; -#endif - return true; -} diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 685ff5ca0..95e780b1d 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -54,6 +54,11 @@ func throwinit() { // The compiler turns a defer statement into a call to this. //go:nosplit func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn + if getg().m.curg != getg() { + // go code on the system stack can't defer + gothrow("defer on system stack") + } + // the arguments of fn are in a perilous state. The stack map // for deferproc does not describe them. So we can't let garbage // collection or stack copying trigger until we've copied them out @@ -61,23 +66,21 @@ func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn // we can only call nosplit routines. argp := uintptr(unsafe.Pointer(&fn)) argp += unsafe.Sizeof(fn) - if GOARCH == "arm" { + if GOARCH == "arm" || GOARCH == "power64" || GOARCH == "power64le" { argp += ptrSize // skip caller's saved link register } - mp := acquirem() - mp.scalararg[0] = uintptr(siz) - mp.ptrarg[0] = unsafe.Pointer(fn) - mp.scalararg[1] = argp - mp.scalararg[2] = getcallerpc(unsafe.Pointer(&siz)) - - if mp.curg != getg() { - // go code on the m stack can't defer - gothrow("defer on m") - } + callerpc := getcallerpc(unsafe.Pointer(&siz)) - onM(deferproc_m) - - releasem(mp) + systemstack(func() { + d := newdefer(siz) + if d._panic != nil { + gothrow("deferproc: d.panic != nil after newdefer") + } + d.fn = fn + d.pc = callerpc + d.argp = argp + memmove(add(unsafe.Pointer(d), unsafe.Sizeof(*d)), unsafe.Pointer(argp), uintptr(siz)) + }) // deferproc returns 0 normally. // a deferred func that stops a panic @@ -159,7 +162,7 @@ func init() { // Allocate a Defer, usually using per-P pool. // Each defer must be released with freedefer. -// Note: runs on M stack +// Note: runs on g0 stack func newdefer(siz int32) *_defer { var d *_defer sc := deferclass(uintptr(siz)) @@ -298,8 +301,6 @@ func Goexit() { goexit() } -func canpanic(*g) bool - // Print all currently active panics. Used when crashing. func printpanics(p *_panic) { if p.link != nil { @@ -318,7 +319,10 @@ func printpanics(p *_panic) { func gopanic(e interface{}) { gp := getg() if gp.m.curg != gp { - gothrow("panic on m stack") + print("panic: ") + printany(e) + print("\n") + gothrow("panic on system stack") } // m.softfloat is set during software floating point. @@ -414,7 +418,7 @@ func gopanic(e interface{}) { // Pass information about recovering frame to recovery. gp.sigcode0 = uintptr(argp) gp.sigcode1 = pc - mcall(recovery_m) + mcall(recovery) gothrow("recovery failed") // mcall should not return } } @@ -466,17 +470,17 @@ func gorecover(argp uintptr) interface{} { //go:nosplit func startpanic() { - onM_signalok(startpanic_m) + systemstack(startpanic_m) } //go:nosplit func dopanic(unused int) { + pc := getcallerpc(unsafe.Pointer(&unused)) + sp := getcallersp(unsafe.Pointer(&unused)) gp := getg() - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(gp) - mp.scalararg[0] = getcallerpc((unsafe.Pointer)(&unused)) - mp.scalararg[1] = getcallersp((unsafe.Pointer)(&unused)) - onM_signalok(dopanic_m) // should never return + systemstack(func() { + dopanic_m(gp, pc, sp) // should never return + }) *(*int)(nil) = 0 } @@ -494,12 +498,12 @@ func throw(s *byte) { //go:nosplit func gothrow(s string) { + print("fatal error: ", s, "\n") gp := getg() if gp.m.throwing == 0 { gp.m.throwing = 1 } startpanic() - print("fatal error: ", s, "\n") dopanic(0) *(*int)(nil) = 0 // not reached } diff --git a/src/runtime/panic1.go b/src/runtime/panic1.go new file mode 100644 index 000000000..17379f963 --- /dev/null +++ b/src/runtime/panic1.go @@ -0,0 +1,161 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Code related to defer, panic and recover. +// TODO: Merge into panic.go. + +//uint32 runtime·panicking; +var paniclk mutex + +const hasLinkRegister = GOARCH == "arm" || GOARCH == "power64" || GOARCH == "power64le" + +// Unwind the stack after a deferred function calls recover +// after a panic. Then arrange to continue running as though +// the caller of the deferred function returned normally. +func recovery(gp *g) { + // Info about defer passed in G struct. + argp := (unsafe.Pointer)(gp.sigcode0) + pc := uintptr(gp.sigcode1) + + // d's arguments need to be in the stack. + if argp != nil && (uintptr(argp) < gp.stack.lo || gp.stack.hi < uintptr(argp)) { + print("recover: ", argp, " not in [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n") + gothrow("bad recovery") + } + + // Make the deferproc for this d return again, + // this time returning 1. The calling function will + // jump to the standard return epilogue. + // The -2*sizeof(uintptr) makes up for the + // two extra words that are on the stack at + // each call to deferproc. + // (The pc we're returning to does pop pop + // before it tests the return value.) + // On the arm and power there are 2 saved LRs mixed in too. + if hasLinkRegister { + gp.sched.sp = uintptr(argp) - 4*ptrSize + } else { + gp.sched.sp = uintptr(argp) - 2*ptrSize + } + gp.sched.pc = pc + gp.sched.lr = 0 + gp.sched.ret = 1 + gogo(&gp.sched) +} + +func startpanic_m() { + _g_ := getg() + if mheap_.cachealloc.size == 0 { // very early + print("runtime: panic before malloc heap initialized\n") + _g_.m.mallocing = 1 // tell rest of panic not to try to malloc + } else if _g_.m.mcache == nil { // can happen if called from signal handler or throw + _g_.m.mcache = allocmcache() + } + + switch _g_.m.dying { + case 0: + _g_.m.dying = 1 + if _g_ != nil { + _g_.writebuf = nil + } + xadd(&panicking, 1) + lock(&paniclk) + if debug.schedtrace > 0 || debug.scheddetail > 0 { + schedtrace(true) + } + freezetheworld() + return + case 1: + // Something failed while panicing, probably the print of the + // argument to panic(). Just print a stack trace and exit. + _g_.m.dying = 2 + print("panic during panic\n") + dopanic(0) + exit(3) + fallthrough + case 2: + // This is a genuine bug in the runtime, we couldn't even + // print the stack trace successfully. + _g_.m.dying = 3 + print("stack trace unavailable\n") + exit(4) + fallthrough + default: + // Can't even print! Just exit. + exit(5) + } +} + +var didothers bool +var deadlock mutex + +func dopanic_m(gp *g, pc, sp uintptr) { + if gp.sig != 0 { + print("[signal ", hex(gp.sig), " code=", hex(gp.sigcode0), " addr=", hex(gp.sigcode1), " pc=", hex(gp.sigpc), "]\n") + } + + var docrash bool + _g_ := getg() + if t := gotraceback(&docrash); t > 0 { + if gp != gp.m.g0 { + print("\n") + goroutineheader(gp) + traceback(pc, sp, 0, gp) + } else if t >= 2 || _g_.m.throwing > 0 { + print("\nruntime stack:\n") + traceback(pc, sp, 0, gp) + } + if !didothers { + didothers = true + tracebackothers(gp) + } + } + unlock(&paniclk) + + if xadd(&panicking, -1) != 0 { + // Some other m is panicking too. + // Let it print what it needs to print. + // Wait forever without chewing up cpu. + // It will exit when it's done. + lock(&deadlock) + lock(&deadlock) + } + + if docrash { + crash() + } + + exit(2) +} + +//go:nosplit +func canpanic(gp *g) bool { + // Note that g is m->gsignal, different from gp. + // Note also that g->m can change at preemption, so m can go stale + // if this function ever makes a function call. + _g_ := getg() + _m_ := _g_.m + + // Is it okay for gp to panic instead of crashing the program? + // Yes, as long as it is running Go code, not runtime code, + // and not stuck in a system call. + if gp == nil || gp != _m_.curg { + return false + } + if _m_.locks-_m_.softfloat != 0 || _m_.mallocing != 0 || _m_.throwing != 0 || _m_.gcing != 0 || _m_.dying != 0 { + return false + } + status := readgstatus(gp) + if status&^_Gscan != _Grunning || gp.syscallsp != 0 { + return false + } + if GOOS == "windows" && _m_.libcallsp != 0 { + return false + } + return true +} diff --git a/src/runtime/parfor.c b/src/runtime/parfor.c deleted file mode 100644 index e44956840..000000000 --- a/src/runtime/parfor.c +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parallel for algorithm. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -struct ParForThread -{ - // the thread's iteration space [32lsb, 32msb) - uint64 pos; - // stats - uint64 nsteal; - uint64 nstealcnt; - uint64 nprocyield; - uint64 nosyield; - uint64 nsleep; - byte pad[CacheLineSize]; -}; - -void -runtime·parforsetup(ParFor *desc, uint32 nthr, uint32 n, void *ctx, bool wait, void (*body)(ParFor*, uint32)) -{ - uint32 i, begin, end; - uint64 *pos; - - if(desc == nil || nthr == 0 || nthr > desc->nthrmax || body == nil) { - runtime·printf("desc=%p nthr=%d count=%d body=%p\n", desc, nthr, n, body); - runtime·throw("parfor: invalid args"); - } - - desc->body = body; - desc->done = 0; - desc->nthr = nthr; - desc->thrseq = 0; - desc->cnt = n; - desc->ctx = ctx; - desc->wait = wait; - desc->nsteal = 0; - desc->nstealcnt = 0; - desc->nprocyield = 0; - desc->nosyield = 0; - desc->nsleep = 0; - for(i=0; i<nthr; i++) { - begin = (uint64)n*i / nthr; - end = (uint64)n*(i+1) / nthr; - pos = &desc->thr[i].pos; - if(((uintptr)pos & 7) != 0) - runtime·throw("parforsetup: pos is not aligned"); - *pos = (uint64)begin | (((uint64)end)<<32); - } -} - -void -runtime·parfordo(ParFor *desc) -{ - ParForThread *me; - uint32 tid, begin, end, begin2, try, victim, i; - uint64 *mypos, *victimpos, pos, newpos; - void (*body)(ParFor*, uint32); - bool idle; - - // Obtain 0-based thread index. - tid = runtime·xadd(&desc->thrseq, 1) - 1; - if(tid >= desc->nthr) { - runtime·printf("tid=%d nthr=%d\n", tid, desc->nthr); - runtime·throw("parfor: invalid tid"); - } - - // If single-threaded, just execute the for serially. - if(desc->nthr==1) { - for(i=0; i<desc->cnt; i++) - desc->body(desc, i); - return; - } - - body = desc->body; - me = &desc->thr[tid]; - mypos = &me->pos; - for(;;) { - for(;;) { - // While there is local work, - // bump low index and execute the iteration. - pos = runtime·xadd64(mypos, 1); - begin = (uint32)pos-1; - end = (uint32)(pos>>32); - if(begin < end) { - body(desc, begin); - continue; - } - break; - } - - // Out of work, need to steal something. - idle = false; - for(try=0;; try++) { - // If we don't see any work for long enough, - // increment the done counter... - if(try > desc->nthr*4 && !idle) { - idle = true; - runtime·xadd(&desc->done, 1); - } - // ...if all threads have incremented the counter, - // we are done. - if(desc->done + !idle == desc->nthr) { - if(!idle) - runtime·xadd(&desc->done, 1); - goto exit; - } - // Choose a random victim for stealing. - victim = runtime·fastrand1() % (desc->nthr-1); - if(victim >= tid) - victim++; - victimpos = &desc->thr[victim].pos; - for(;;) { - // See if it has any work. - pos = runtime·atomicload64(victimpos); - begin = (uint32)pos; - end = (uint32)(pos>>32); - if(begin+1 >= end) { - begin = end = 0; - break; - } - if(idle) { - runtime·xadd(&desc->done, -1); - idle = false; - } - begin2 = begin + (end-begin)/2; - newpos = (uint64)begin | (uint64)begin2<<32; - if(runtime·cas64(victimpos, pos, newpos)) { - begin = begin2; - break; - } - } - if(begin < end) { - // Has successfully stolen some work. - if(idle) - runtime·throw("parfor: should not be idle"); - runtime·atomicstore64(mypos, (uint64)begin | (uint64)end<<32); - me->nsteal++; - me->nstealcnt += end-begin; - break; - } - // Backoff. - if(try < desc->nthr) { - // nothing - } else if (try < 4*desc->nthr) { - me->nprocyield++; - runtime·procyield(20); - // If a caller asked not to wait for the others, exit now - // (assume that most work is already done at this point). - } else if (!desc->wait) { - if(!idle) - runtime·xadd(&desc->done, 1); - goto exit; - } else if (try < 6*desc->nthr) { - me->nosyield++; - runtime·osyield(); - } else { - me->nsleep++; - runtime·usleep(1); - } - } - } -exit: - runtime·xadd64(&desc->nsteal, me->nsteal); - runtime·xadd64(&desc->nstealcnt, me->nstealcnt); - runtime·xadd64(&desc->nprocyield, me->nprocyield); - runtime·xadd64(&desc->nosyield, me->nosyield); - runtime·xadd64(&desc->nsleep, me->nsleep); - me->nsteal = 0; - me->nstealcnt = 0; - me->nprocyield = 0; - me->nosyield = 0; - me->nsleep = 0; -} - -// For testing from Go. -void -runtime·newparfor_m(void) -{ - g->m->ptrarg[0] = runtime·parforalloc(g->m->scalararg[0]); -} - -void -runtime·parforsetup_m(void) -{ - ParFor *desc; - void *ctx; - void (*body)(ParFor*, uint32); - - desc = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - ctx = g->m->ptrarg[1]; - g->m->ptrarg[1] = nil; - body = g->m->ptrarg[2]; - g->m->ptrarg[2] = nil; - - runtime·parforsetup(desc, g->m->scalararg[0], g->m->scalararg[1], ctx, g->m->scalararg[2], body); -} - -void -runtime·parfordo_m(void) -{ - ParFor *desc; - - desc = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - runtime·parfordo(desc); -} - -void -runtime·parforiters_m(void) -{ - ParFor *desc; - uintptr tid; - - desc = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - tid = g->m->scalararg[0]; - g->m->scalararg[0] = desc->thr[tid].pos; - g->m->scalararg[1] = desc->thr[tid].pos>>32; -} diff --git a/src/runtime/parfor.go b/src/runtime/parfor.go new file mode 100644 index 000000000..14870c9fe --- /dev/null +++ b/src/runtime/parfor.go @@ -0,0 +1,186 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Parallel for algorithm. + +package runtime + +import "unsafe" + +type parforthread struct { + // the thread's iteration space [32lsb, 32msb) + pos uint64 + // stats + nsteal uint64 + nstealcnt uint64 + nprocyield uint64 + nosyield uint64 + nsleep uint64 + pad [_CacheLineSize]byte +} + +func desc_thr_index(desc *parfor, i uint32) *parforthread { + return (*parforthread)(add(unsafe.Pointer(desc.thr), uintptr(i)*unsafe.Sizeof(*desc.thr))) +} + +func parforsetup(desc *parfor, nthr, n uint32, ctx unsafe.Pointer, wait bool, body func(*parfor, uint32)) { + if desc == nil || nthr == 0 || nthr > desc.nthrmax || body == nil { + print("desc=", desc, " nthr=", nthr, " count=", n, " body=", body, "\n") + gothrow("parfor: invalid args") + } + + desc.body = *(*unsafe.Pointer)(unsafe.Pointer(&body)) + desc.done = 0 + desc.nthr = nthr + desc.thrseq = 0 + desc.cnt = n + desc.ctx = ctx + desc.wait = wait + desc.nsteal = 0 + desc.nstealcnt = 0 + desc.nprocyield = 0 + desc.nosyield = 0 + desc.nsleep = 0 + + for i := uint32(0); i < nthr; i++ { + begin := uint32(uint64(n) * uint64(i) / uint64(nthr)) + end := uint32(uint64(n) * uint64(i+1) / uint64(nthr)) + pos := &desc_thr_index(desc, i).pos + if uintptr(unsafe.Pointer(pos))&7 != 0 { + gothrow("parforsetup: pos is not aligned") + } + *pos = uint64(begin) | uint64(end)<<32 + } +} + +func parfordo(desc *parfor) { + // Obtain 0-based thread index. + tid := xadd(&desc.thrseq, 1) - 1 + if tid >= desc.nthr { + print("tid=", tid, " nthr=", desc.nthr, "\n") + gothrow("parfor: invalid tid") + } + + // If single-threaded, just execute the for serially. + body := *(*func(*parfor, uint32))(unsafe.Pointer(&desc.body)) + if desc.nthr == 1 { + for i := uint32(0); i < desc.cnt; i++ { + body(desc, i) + } + return + } + + me := desc_thr_index(desc, tid) + mypos := &me.pos + for { + for { + // While there is local work, + // bump low index and execute the iteration. + pos := xadd64(mypos, 1) + begin := uint32(pos) - 1 + end := uint32(pos >> 32) + if begin < end { + body(desc, begin) + continue + } + break + } + + // Out of work, need to steal something. + idle := false + for try := uint32(0); ; try++ { + // If we don't see any work for long enough, + // increment the done counter... + if try > desc.nthr*4 && !idle { + idle = true + xadd(&desc.done, 1) + } + + // ...if all threads have incremented the counter, + // we are done. + extra := uint32(0) + if !idle { + extra = 1 + } + if desc.done+extra == desc.nthr { + if !idle { + xadd(&desc.done, 1) + } + goto exit + } + + // Choose a random victim for stealing. + var begin, end uint32 + victim := fastrand1() % (desc.nthr - 1) + if victim >= tid { + victim++ + } + victimpos := &desc_thr_index(desc, victim).pos + for { + // See if it has any work. + pos := atomicload64(victimpos) + begin = uint32(pos) + end = uint32(pos >> 32) + if begin+1 >= end { + end = 0 + begin = end + break + } + if idle { + xadd(&desc.done, -1) + idle = false + } + begin2 := begin + (end-begin)/2 + newpos := uint64(begin) | uint64(begin2)<<32 + if cas64(victimpos, pos, newpos) { + begin = begin2 + break + } + } + if begin < end { + // Has successfully stolen some work. + if idle { + gothrow("parfor: should not be idle") + } + atomicstore64(mypos, uint64(begin)|uint64(end)<<32) + me.nsteal++ + me.nstealcnt += uint64(end) - uint64(begin) + break + } + + // Backoff. + if try < desc.nthr { + // nothing + } else if try < 4*desc.nthr { + me.nprocyield++ + procyield(20) + } else if !desc.wait { + // If a caller asked not to wait for the others, exit now + // (assume that most work is already done at this point). + if !idle { + xadd(&desc.done, 1) + } + goto exit + } else if try < 6*desc.nthr { + me.nosyield++ + osyield() + } else { + me.nsleep++ + usleep(1) + } + } + } + +exit: + xadd64(&desc.nsteal, int64(me.nsteal)) + xadd64(&desc.nstealcnt, int64(me.nstealcnt)) + xadd64(&desc.nprocyield, int64(me.nprocyield)) + xadd64(&desc.nosyield, int64(me.nosyield)) + xadd64(&desc.nsleep, int64(me.nsleep)) + me.nsteal = 0 + me.nstealcnt = 0 + me.nprocyield = 0 + me.nosyield = 0 + me.nsleep = 0 +} diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 8677cb30c..101c05989 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -249,7 +249,7 @@ func TestGoroutineSwitch(t *testing.T) { // exists to record a PC without a traceback. Those are okay. if len(stk) == 2 { f := runtime.FuncForPC(stk[1]) - if f != nil && (f.Name() == "System" || f.Name() == "ExternalCode" || f.Name() == "GC") { + if f != nil && (f.Name() == "runtime._System" || f.Name() == "runtime._ExternalCode" || f.Name() == "runtime._GC") { return } } diff --git a/src/runtime/proc.c b/src/runtime/proc.c deleted file mode 100644 index 91e3fe16d..000000000 --- a/src/runtime/proc.c +++ /dev/null @@ -1,3489 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "zaexperiment.h" -#include "malloc.h" -#include "stack.h" -#include "race.h" -#include "type.h" -#include "mgc0.h" -#include "textflag.h" - -// Goroutine scheduler -// The scheduler's job is to distribute ready-to-run goroutines over worker threads. -// -// The main concepts are: -// G - goroutine. -// M - worker thread, or machine. -// P - processor, a resource that is required to execute Go code. -// M must have an associated P to execute Go code, however it can be -// blocked or in a syscall w/o an associated P. -// -// Design doc at http://golang.org/s/go11sched. - -enum -{ - // Number of goroutine ids to grab from runtime·sched.goidgen to local per-P cache at once. - // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. - GoidCacheBatch = 16, -}; - -SchedT runtime·sched; -int32 runtime·gomaxprocs; -uint32 runtime·needextram; -bool runtime·iscgo; -M runtime·m0; -G runtime·g0; // idle goroutine for m0 -G* runtime·lastg; -M* runtime·allm; -M* runtime·extram; -P* runtime·allp[MaxGomaxprocs+1]; -int8* runtime·goos; -int32 runtime·ncpu; -int32 runtime·newprocs; - -Mutex runtime·allglock; // the following vars are protected by this lock or by stoptheworld -G** runtime·allg; -Slice runtime·allgs; -uintptr runtime·allglen; -ForceGCState runtime·forcegc; - -void runtime·mstart(void); -static void runqput(P*, G*); -static G* runqget(P*); -static bool runqputslow(P*, G*, uint32, uint32); -static G* runqsteal(P*, P*); -static void mput(M*); -static M* mget(void); -static void mcommoninit(M*); -static void schedule(void); -static void procresize(int32); -static void acquirep(P*); -static P* releasep(void); -static void newm(void(*)(void), P*); -static void stopm(void); -static void startm(P*, bool); -static void handoffp(P*); -static void wakep(void); -static void stoplockedm(void); -static void startlockedm(G*); -static void sysmon(void); -static uint32 retake(int64); -static void incidlelocked(int32); -static void checkdead(void); -static void exitsyscall0(G*); -void runtime·park_m(G*); -static void goexit0(G*); -static void gfput(P*, G*); -static G* gfget(P*); -static void gfpurge(P*); -static void globrunqput(G*); -static void globrunqputbatch(G*, G*, int32); -static G* globrunqget(P*, int32); -static P* pidleget(void); -static void pidleput(P*); -static void injectglist(G*); -static bool preemptall(void); -static bool preemptone(P*); -static bool exitsyscallfast(void); -static bool haveexperiment(int8*); -void runtime·allgadd(G*); -static void dropg(void); - -extern String runtime·buildVersion; - -// For cgo-using programs with external linking, -// export "main" (defined in assembly) so that libc can handle basic -// C runtime startup and call the Go program as if it were -// the C main function. -#pragma cgo_export_static main - -// Filled in by dynamic linker when Cgo is available. -void (*_cgo_init)(void); -void (*_cgo_malloc)(void); -void (*_cgo_free)(void); - -// Copy for Go code. -void* runtime·cgoMalloc; -void* runtime·cgoFree; - -// The bootstrap sequence is: -// -// call osinit -// call schedinit -// make & queue new G -// call runtime·mstart -// -// The new G calls runtime·main. -void -runtime·schedinit(void) -{ - int32 n, procs; - byte *p; - - // raceinit must be the first call to race detector. - // In particular, it must be done before mallocinit below calls racemapshadow. - if(raceenabled) - g->racectx = runtime·raceinit(); - - runtime·sched.maxmcount = 10000; - - runtime·tracebackinit(); - runtime·symtabinit(); - runtime·stackinit(); - runtime·mallocinit(); - mcommoninit(g->m); - - runtime·goargs(); - runtime·goenvs(); - runtime·parsedebugvars(); - runtime·gcinit(); - - runtime·sched.lastpoll = runtime·nanotime(); - procs = 1; - p = runtime·getenv("GOMAXPROCS"); - if(p != nil && (n = runtime·atoi(p)) > 0) { - if(n > MaxGomaxprocs) - n = MaxGomaxprocs; - procs = n; - } - procresize(procs); - - if(runtime·buildVersion.str == nil) { - // Condition should never trigger. This code just serves - // to ensure runtime·buildVersion is kept in the resulting binary. - runtime·buildVersion.str = (uint8*)"unknown"; - runtime·buildVersion.len = 7; - } - - runtime·cgoMalloc = _cgo_malloc; - runtime·cgoFree = _cgo_free; -} - -void -runtime·newsysmon(void) -{ - newm(sysmon, nil); -} - -static void -dumpgstatus(G* gp) -{ - runtime·printf("runtime: gp: gp=%p, goid=%D, gp->atomicstatus=%x\n", gp, gp->goid, runtime·readgstatus(gp)); - runtime·printf("runtime: g: g=%p, goid=%D, g->atomicstatus=%x\n", g, g->goid, runtime·readgstatus(g)); -} - -static void -checkmcount(void) -{ - // sched lock is held - if(runtime·sched.mcount > runtime·sched.maxmcount){ - runtime·printf("runtime: program exceeds %d-thread limit\n", runtime·sched.maxmcount); - runtime·throw("thread exhaustion"); - } -} - -static void -mcommoninit(M *mp) -{ - // g0 stack won't make sense for user (and is not necessary unwindable). - if(g != g->m->g0) - runtime·callers(1, mp->createstack, nelem(mp->createstack)); - - mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); - - runtime·lock(&runtime·sched.lock); - mp->id = runtime·sched.mcount++; - checkmcount(); - runtime·mpreinit(mp); - if(mp->gsignal) - mp->gsignal->stackguard1 = mp->gsignal->stack.lo + StackGuard; - - // Add to runtime·allm so garbage collector doesn't free g->m - // when it is just in a register or thread-local storage. - mp->alllink = runtime·allm; - // runtime·NumCgoCall() iterates over allm w/o schedlock, - // so we need to publish it safely. - runtime·atomicstorep(&runtime·allm, mp); - runtime·unlock(&runtime·sched.lock); -} - -// Mark gp ready to run. -void -runtime·ready(G *gp) -{ - uint32 status; - - status = runtime·readgstatus(gp); - // Mark runnable. - g->m->locks++; // disable preemption because it can be holding p in a local var - if((status&~Gscan) != Gwaiting){ - dumpgstatus(gp); - runtime·throw("bad g->status in ready"); - } - // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq - runtime·casgstatus(gp, Gwaiting, Grunnable); - runqput(g->m->p, gp); - if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0) // TODO: fast atomic - wakep(); - g->m->locks--; - if(g->m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; -} - -void -runtime·ready_m(void) -{ - G *gp; - - gp = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - runtime·ready(gp); -} - -int32 -runtime·gcprocs(void) -{ - int32 n; - - // Figure out how many CPUs to use during GC. - // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. - runtime·lock(&runtime·sched.lock); - n = runtime·gomaxprocs; - if(n > runtime·ncpu) - n = runtime·ncpu; - if(n > MaxGcproc) - n = MaxGcproc; - if(n > runtime·sched.nmidle+1) // one M is currently running - n = runtime·sched.nmidle+1; - runtime·unlock(&runtime·sched.lock); - return n; -} - -static bool -needaddgcproc(void) -{ - int32 n; - - runtime·lock(&runtime·sched.lock); - n = runtime·gomaxprocs; - if(n > runtime·ncpu) - n = runtime·ncpu; - if(n > MaxGcproc) - n = MaxGcproc; - n -= runtime·sched.nmidle+1; // one M is currently running - runtime·unlock(&runtime·sched.lock); - return n > 0; -} - -void -runtime·helpgc(int32 nproc) -{ - M *mp; - int32 n, pos; - - runtime·lock(&runtime·sched.lock); - pos = 0; - for(n = 1; n < nproc; n++) { // one M is currently running - if(runtime·allp[pos]->mcache == g->m->mcache) - pos++; - mp = mget(); - if(mp == nil) - runtime·throw("runtime·gcprocs inconsistency"); - mp->helpgc = n; - mp->mcache = runtime·allp[pos]->mcache; - pos++; - runtime·notewakeup(&mp->park); - } - runtime·unlock(&runtime·sched.lock); -} - -// Similar to stoptheworld but best-effort and can be called several times. -// There is no reverse operation, used during crashing. -// This function must not lock any mutexes. -void -runtime·freezetheworld(void) -{ - int32 i; - - if(runtime·gomaxprocs == 1) - return; - // stopwait and preemption requests can be lost - // due to races with concurrently executing threads, - // so try several times - for(i = 0; i < 5; i++) { - // this should tell the scheduler to not start any new goroutines - runtime·sched.stopwait = 0x7fffffff; - runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1); - // this should stop running goroutines - if(!preemptall()) - break; // no running goroutines - runtime·usleep(1000); - } - // to be sure - runtime·usleep(1000); - preemptall(); - runtime·usleep(1000); -} - -static bool -isscanstatus(uint32 status) -{ - if(status == Gscan) - runtime·throw("isscanstatus: Bad status Gscan"); - return (status&Gscan) == Gscan; -} - -// All reads and writes of g's status go through readgstatus, casgstatus -// castogscanstatus, casfromgscanstatus. -#pragma textflag NOSPLIT -uint32 -runtime·readgstatus(G *gp) -{ - return runtime·atomicload(&gp->atomicstatus); -} - -// The Gscanstatuses are acting like locks and this releases them. -// If it proves to be a performance hit we should be able to make these -// simple atomic stores but for now we are going to throw if -// we see an inconsistent state. -void -runtime·casfromgscanstatus(G *gp, uint32 oldval, uint32 newval) -{ - bool success = false; - - // Check that transition is valid. - switch(oldval) { - case Gscanrunnable: - case Gscanwaiting: - case Gscanrunning: - case Gscansyscall: - if(newval == (oldval&~Gscan)) - success = runtime·cas(&gp->atomicstatus, oldval, newval); - break; - case Gscanenqueue: - if(newval == Gwaiting) - success = runtime·cas(&gp->atomicstatus, oldval, newval); - break; - } - if(!success){ - runtime·printf("runtime: casfromgscanstatus failed gp=%p, oldval=%d, newval=%d\n", - gp, oldval, newval); - dumpgstatus(gp); - runtime·throw("casfromgscanstatus: gp->status is not in scan state"); - } -} - -// This will return false if the gp is not in the expected status and the cas fails. -// This acts like a lock acquire while the casfromgstatus acts like a lock release. -bool -runtime·castogscanstatus(G *gp, uint32 oldval, uint32 newval) -{ - switch(oldval) { - case Grunnable: - case Gwaiting: - case Gsyscall: - if(newval == (oldval|Gscan)) - return runtime·cas(&gp->atomicstatus, oldval, newval); - break; - case Grunning: - if(newval == Gscanrunning || newval == Gscanenqueue) - return runtime·cas(&gp->atomicstatus, oldval, newval); - break; - } - - runtime·printf("runtime: castogscanstatus oldval=%d newval=%d\n", oldval, newval); - runtime·throw("castogscanstatus"); - return false; // not reached -} - -static void badcasgstatus(void); -static void helpcasgstatus(void); - -// If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus -// and casfromgscanstatus instead. -// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that -// put it in the Gscan state is finished. -#pragma textflag NOSPLIT -void -runtime·casgstatus(G *gp, uint32 oldval, uint32 newval) -{ - void (*fn)(void); - - if((oldval&Gscan) || (newval&Gscan) || oldval == newval) { - g->m->scalararg[0] = oldval; - g->m->scalararg[1] = newval; - fn = badcasgstatus; - runtime·onM(&fn); - } - - // loop if gp->atomicstatus is in a scan state giving - // GC time to finish and change the state to oldval. - while(!runtime·cas(&gp->atomicstatus, oldval, newval)) { - // Help GC if needed. - if(gp->preemptscan && !gp->gcworkdone && (oldval == Grunning || oldval == Gsyscall)) { - gp->preemptscan = false; - g->m->ptrarg[0] = gp; - fn = helpcasgstatus; - runtime·onM(&fn); - } - } -} - -static void -badcasgstatus(void) -{ - uint32 oldval, newval; - - oldval = g->m->scalararg[0]; - newval = g->m->scalararg[1]; - g->m->scalararg[0] = 0; - g->m->scalararg[1] = 0; - - runtime·printf("casgstatus: oldval=%d, newval=%d\n", oldval, newval); - runtime·throw("casgstatus: bad incoming values"); -} - -static void -helpcasgstatus(void) -{ - G *gp; - - gp = g->m->ptrarg[0]; - g->m->ptrarg[0] = 0; - runtime·gcphasework(gp); -} - -// stopg ensures that gp is stopped at a GC safe point where its stack can be scanned -// or in the context of a moving collector the pointers can be flipped from pointing -// to old object to pointing to new objects. -// If stopg returns true, the caller knows gp is at a GC safe point and will remain there until -// the caller calls restartg. -// If stopg returns false, the caller is not responsible for calling restartg. This can happen -// if another thread, either the gp itself or another GC thread is taking the responsibility -// to do the GC work related to this thread. -bool -runtime·stopg(G *gp) -{ - uint32 s; - - for(;;) { - if(gp->gcworkdone) - return false; - - s = runtime·readgstatus(gp); - switch(s) { - default: - dumpgstatus(gp); - runtime·throw("stopg: gp->atomicstatus is not valid"); - - case Gdead: - return false; - - case Gcopystack: - // Loop until a new stack is in place. - break; - - case Grunnable: - case Gsyscall: - case Gwaiting: - // Claim goroutine by setting scan bit. - if(!runtime·castogscanstatus(gp, s, s|Gscan)) - break; - // In scan state, do work. - runtime·gcphasework(gp); - return true; - - case Gscanrunnable: - case Gscanwaiting: - case Gscansyscall: - // Goroutine already claimed by another GC helper. - return false; - - case Grunning: - // Claim goroutine, so we aren't racing with a status - // transition away from Grunning. - if(!runtime·castogscanstatus(gp, Grunning, Gscanrunning)) - break; - - // Mark gp for preemption. - if(!gp->gcworkdone) { - gp->preemptscan = true; - gp->preempt = true; - gp->stackguard0 = StackPreempt; - } - - // Unclaim. - runtime·casfromgscanstatus(gp, Gscanrunning, Grunning); - return false; - } - } - // Should not be here.... -} - -// The GC requests that this routine be moved from a scanmumble state to a mumble state. -void -runtime·restartg (G *gp) -{ - uint32 s; - - s = runtime·readgstatus(gp); - switch(s) { - default: - dumpgstatus(gp); - runtime·throw("restartg: unexpected status"); - - case Gdead: - break; - - case Gscanrunnable: - case Gscanwaiting: - case Gscansyscall: - runtime·casfromgscanstatus(gp, s, s&~Gscan); - break; - - case Gscanenqueue: - // Scan is now completed. - // Goroutine now needs to be made runnable. - // We put it on the global run queue; ready blocks on the global scheduler lock. - runtime·casfromgscanstatus(gp, Gscanenqueue, Gwaiting); - if(gp != g->m->curg) - runtime·throw("processing Gscanenqueue on wrong m"); - dropg(); - runtime·ready(gp); - break; - } -} - -static void -stopscanstart(G* gp) -{ - if(g == gp) - runtime·throw("GC not moved to G0"); - if(runtime·stopg(gp)) { - if(!isscanstatus(runtime·readgstatus(gp))) { - dumpgstatus(gp); - runtime·throw("GC not in scan state"); - } - runtime·restartg(gp); - } -} - -// Runs on g0 and does the actual work after putting the g back on the run queue. -static void -mquiesce(G *gpmaster) -{ - G* gp; - uint32 i; - uint32 status; - uint32 activeglen; - - activeglen = runtime·allglen; - // enqueue the calling goroutine. - runtime·restartg(gpmaster); - for(i = 0; i < activeglen; i++) { - gp = runtime·allg[i]; - if(runtime·readgstatus(gp) == Gdead) - gp->gcworkdone = true; // noop scan. - else - gp->gcworkdone = false; - stopscanstart(gp); - } - - // Check that the G's gcwork (such as scanning) has been done. If not do it now. - // You can end up doing work here if the page trap on a Grunning Goroutine has - // not been sprung or in some race situations. For example a runnable goes dead - // and is started up again with a gp->gcworkdone set to false. - for(i = 0; i < activeglen; i++) { - gp = runtime·allg[i]; - while (!gp->gcworkdone) { - status = runtime·readgstatus(gp); - if(status == Gdead) { - gp->gcworkdone = true; // scan is a noop - break; - //do nothing, scan not needed. - } - if(status == Grunning && gp->stackguard0 == (uintptr)StackPreempt && runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) // nanosecond arg - runtime·noteclear(&runtime·sched.stopnote); - else - stopscanstart(gp); - } - } - - for(i = 0; i < activeglen; i++) { - gp = runtime·allg[i]; - status = runtime·readgstatus(gp); - if(isscanstatus(status)) { - runtime·printf("mstopandscang:bottom: post scan bad status gp=%p has status %x\n", gp, status); - dumpgstatus(gp); - } - if(!gp->gcworkdone && status != Gdead) { - runtime·printf("mstopandscang:bottom: post scan gp=%p->gcworkdone still false\n", gp); - dumpgstatus(gp); - } - } - - schedule(); // Never returns. -} - -// quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point. -// If the global runtime·gcphase is GCmark quiesce will ensure that all of the goroutine's stacks -// have been scanned before it returns. -void -runtime·quiesce(G* mastergp) -{ - void (*fn)(G*); - - runtime·castogscanstatus(mastergp, Grunning, Gscanenqueue); - // Now move this to the g0 (aka m) stack. - // g0 will potentially scan this thread and put mastergp on the runqueue - fn = mquiesce; - runtime·mcall(&fn); -} - -// This is used by the GC as well as the routines that do stack dumps. In the case -// of GC all the routines can be reliably stopped. This is not always the case -// when the system is in panic or being exited. -void -runtime·stoptheworld(void) -{ - int32 i; - uint32 s; - P *p; - bool wait; - - // If we hold a lock, then we won't be able to stop another M - // that is blocked trying to acquire the lock. - if(g->m->locks > 0) - runtime·throw("stoptheworld: holding locks"); - - runtime·lock(&runtime·sched.lock); - runtime·sched.stopwait = runtime·gomaxprocs; - runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1); - preemptall(); - // stop current P - g->m->p->status = Pgcstop; // Pgcstop is only diagnostic. - runtime·sched.stopwait--; - // try to retake all P's in Psyscall status - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - s = p->status; - if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) - runtime·sched.stopwait--; - } - // stop idle P's - while(p = pidleget()) { - p->status = Pgcstop; - runtime·sched.stopwait--; - } - wait = runtime·sched.stopwait > 0; - runtime·unlock(&runtime·sched.lock); - - // wait for remaining P's to stop voluntarily - if(wait) { - for(;;) { - // wait for 100us, then try to re-preempt in case of any races - if(runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) { - runtime·noteclear(&runtime·sched.stopnote); - break; - } - preemptall(); - } - } - if(runtime·sched.stopwait) - runtime·throw("stoptheworld: not stopped"); - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - if(p->status != Pgcstop) - runtime·throw("stoptheworld: not stopped"); - } -} - -static void -mhelpgc(void) -{ - g->m->helpgc = -1; -} - -void -runtime·starttheworld(void) -{ - P *p, *p1; - M *mp; - G *gp; - bool add; - - g->m->locks++; // disable preemption because it can be holding p in a local var - gp = runtime·netpoll(false); // non-blocking - injectglist(gp); - add = needaddgcproc(); - runtime·lock(&runtime·sched.lock); - if(runtime·newprocs) { - procresize(runtime·newprocs); - runtime·newprocs = 0; - } else - procresize(runtime·gomaxprocs); - runtime·sched.gcwaiting = 0; - - p1 = nil; - while(p = pidleget()) { - // procresize() puts p's with work at the beginning of the list. - // Once we reach a p without a run queue, the rest don't have one either. - if(p->runqhead == p->runqtail) { - pidleput(p); - break; - } - p->m = mget(); - p->link = p1; - p1 = p; - } - if(runtime·sched.sysmonwait) { - runtime·sched.sysmonwait = false; - runtime·notewakeup(&runtime·sched.sysmonnote); - } - runtime·unlock(&runtime·sched.lock); - - while(p1) { - p = p1; - p1 = p1->link; - if(p->m) { - mp = p->m; - p->m = nil; - if(mp->nextp) - runtime·throw("starttheworld: inconsistent mp->nextp"); - mp->nextp = p; - runtime·notewakeup(&mp->park); - } else { - // Start M to run P. Do not start another M below. - newm(nil, p); - add = false; - } - } - - if(add) { - // If GC could have used another helper proc, start one now, - // in the hope that it will be available next time. - // It would have been even better to start it before the collection, - // but doing so requires allocating memory, so it's tricky to - // coordinate. This lazy approach works out in practice: - // we don't mind if the first couple gc rounds don't have quite - // the maximum number of procs. - newm(mhelpgc, nil); - } - g->m->locks--; - if(g->m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; -} - -static void mstart(void); - -// Called to start an M. -#pragma textflag NOSPLIT -void -runtime·mstart(void) -{ - uintptr x, size; - - if(g->stack.lo == 0) { - // Initialize stack bounds from system stack. - // Cgo may have left stack size in stack.hi. - size = g->stack.hi; - if(size == 0) - size = 8192; - g->stack.hi = (uintptr)&x; - g->stack.lo = g->stack.hi - size + 1024; - } - - // Initialize stack guards so that we can start calling - // both Go and C functions with stack growth prologues. - g->stackguard0 = g->stack.lo + StackGuard; - g->stackguard1 = g->stackguard0; - mstart(); -} - -static void -mstart(void) -{ - if(g != g->m->g0) - runtime·throw("bad runtime·mstart"); - - // Record top of stack for use by mcall. - // Once we call schedule we're never coming back, - // so other calls can reuse this stack space. - runtime·gosave(&g->m->g0->sched); - g->m->g0->sched.pc = (uintptr)-1; // make sure it is never used - runtime·asminit(); - runtime·minit(); - - // Install signal handlers; after minit so that minit can - // prepare the thread to be able to handle the signals. - if(g->m == &runtime·m0) - runtime·initsig(); - - if(g->m->mstartfn) - g->m->mstartfn(); - - if(g->m->helpgc) { - g->m->helpgc = 0; - stopm(); - } else if(g->m != &runtime·m0) { - acquirep(g->m->nextp); - g->m->nextp = nil; - } - schedule(); - - // TODO(brainman): This point is never reached, because scheduler - // does not release os threads at the moment. But once this path - // is enabled, we must remove our seh here. -} - -// When running with cgo, we call _cgo_thread_start -// to start threads for us so that we can play nicely with -// foreign code. -void (*_cgo_thread_start)(void*); - -typedef struct CgoThreadStart CgoThreadStart; -struct CgoThreadStart -{ - G *g; - uintptr *tls; - void (*fn)(void); -}; - -M *runtime·newM(void); // in proc.go - -// Allocate a new m unassociated with any thread. -// Can use p for allocation context if needed. -M* -runtime·allocm(P *p) -{ - M *mp; - - g->m->locks++; // disable GC because it can be called from sysmon - if(g->m->p == nil) - acquirep(p); // temporarily borrow p for mallocs in this function - mp = runtime·newM(); - mcommoninit(mp); - - // In case of cgo or Solaris, pthread_create will make us a stack. - // Windows and Plan 9 will layout sched stack on OS stack. - if(runtime·iscgo || Solaris || Windows || Plan9) - mp->g0 = runtime·malg(-1); - else - mp->g0 = runtime·malg(8192); - mp->g0->m = mp; - - if(p == g->m->p) - releasep(); - g->m->locks--; - if(g->m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - - return mp; -} - -G *runtime·newG(void); // in proc.go - -static G* -allocg(void) -{ - return runtime·newG(); -} - -static M* lockextra(bool nilokay); -static void unlockextra(M*); - -// needm is called when a cgo callback happens on a -// thread without an m (a thread not created by Go). -// In this case, needm is expected to find an m to use -// and return with m, g initialized correctly. -// Since m and g are not set now (likely nil, but see below) -// needm is limited in what routines it can call. In particular -// it can only call nosplit functions (textflag 7) and cannot -// do any scheduling that requires an m. -// -// In order to avoid needing heavy lifting here, we adopt -// the following strategy: there is a stack of available m's -// that can be stolen. Using compare-and-swap -// to pop from the stack has ABA races, so we simulate -// a lock by doing an exchange (via casp) to steal the stack -// head and replace the top pointer with MLOCKED (1). -// This serves as a simple spin lock that we can use even -// without an m. The thread that locks the stack in this way -// unlocks the stack by storing a valid stack head pointer. -// -// In order to make sure that there is always an m structure -// available to be stolen, we maintain the invariant that there -// is always one more than needed. At the beginning of the -// program (if cgo is in use) the list is seeded with a single m. -// If needm finds that it has taken the last m off the list, its job -// is - once it has installed its own m so that it can do things like -// allocate memory - to create a spare m and put it on the list. -// -// Each of these extra m's also has a g0 and a curg that are -// pressed into service as the scheduling stack and current -// goroutine for the duration of the cgo callback. -// -// When the callback is done with the m, it calls dropm to -// put the m back on the list. -#pragma textflag NOSPLIT -void -runtime·needm(byte x) -{ - M *mp; - - if(runtime·needextram) { - // Can happen if C/C++ code calls Go from a global ctor. - // Can not throw, because scheduler is not initialized yet. - runtime·write(2, "fatal error: cgo callback before cgo call\n", - sizeof("fatal error: cgo callback before cgo call\n")-1); - runtime·exit(1); - } - - // Lock extra list, take head, unlock popped list. - // nilokay=false is safe here because of the invariant above, - // that the extra list always contains or will soon contain - // at least one m. - mp = lockextra(false); - - // Set needextram when we've just emptied the list, - // so that the eventual call into cgocallbackg will - // allocate a new m for the extra list. We delay the - // allocation until then so that it can be done - // after exitsyscall makes sure it is okay to be - // running at all (that is, there's no garbage collection - // running right now). - mp->needextram = mp->schedlink == nil; - unlockextra(mp->schedlink); - - // Install g (= m->g0) and set the stack bounds - // to match the current stack. We don't actually know - // how big the stack is, like we don't know how big any - // scheduling stack is, but we assume there's at least 32 kB, - // which is more than enough for us. - runtime·setg(mp->g0); - g->stack.hi = (uintptr)(&x + 1024); - g->stack.lo = (uintptr)(&x - 32*1024); - g->stackguard0 = g->stack.lo + StackGuard; - - // Initialize this thread to use the m. - runtime·asminit(); - runtime·minit(); -} - -// newextram allocates an m and puts it on the extra list. -// It is called with a working local m, so that it can do things -// like call schedlock and allocate. -void -runtime·newextram(void) -{ - M *mp, *mnext; - G *gp; - - // Create extra goroutine locked to extra m. - // The goroutine is the context in which the cgo callback will run. - // The sched.pc will never be returned to, but setting it to - // runtime.goexit makes clear to the traceback routines where - // the goroutine stack ends. - mp = runtime·allocm(nil); - gp = runtime·malg(4096); - gp->sched.pc = (uintptr)runtime·goexit + PCQuantum; - gp->sched.sp = gp->stack.hi; - gp->sched.sp -= 4*sizeof(uintreg); // extra space in case of reads slightly beyond frame - gp->sched.lr = 0; - gp->sched.g = gp; - gp->syscallpc = gp->sched.pc; - gp->syscallsp = gp->sched.sp; - // malg returns status as Gidle, change to Gsyscall before adding to allg - // where GC will see it. - runtime·casgstatus(gp, Gidle, Gsyscall); - gp->m = mp; - mp->curg = gp; - mp->locked = LockInternal; - mp->lockedg = gp; - gp->lockedm = mp; - gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); - if(raceenabled) - gp->racectx = runtime·racegostart(runtime·newextram); - // put on allg for garbage collector - runtime·allgadd(gp); - - // Add m to the extra list. - mnext = lockextra(true); - mp->schedlink = mnext; - unlockextra(mp); -} - -// dropm is called when a cgo callback has called needm but is now -// done with the callback and returning back into the non-Go thread. -// It puts the current m back onto the extra list. -// -// The main expense here is the call to signalstack to release the -// m's signal stack, and then the call to needm on the next callback -// from this thread. It is tempting to try to save the m for next time, -// which would eliminate both these costs, but there might not be -// a next time: the current thread (which Go does not control) might exit. -// If we saved the m for that thread, there would be an m leak each time -// such a thread exited. Instead, we acquire and release an m on each -// call. These should typically not be scheduling operations, just a few -// atomics, so the cost should be small. -// -// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread -// variable using pthread_key_create. Unlike the pthread keys we already use -// on OS X, this dummy key would never be read by Go code. It would exist -// only so that we could register at thread-exit-time destructor. -// That destructor would put the m back onto the extra list. -// This is purely a performance optimization. The current version, -// in which dropm happens on each cgo call, is still correct too. -// We may have to keep the current version on systems with cgo -// but without pthreads, like Windows. -void -runtime·dropm(void) -{ - M *mp, *mnext; - - // Undo whatever initialization minit did during needm. - runtime·unminit(); - - // Clear m and g, and return m to the extra list. - // After the call to setmg we can only call nosplit functions. - mp = g->m; - runtime·setg(nil); - - mnext = lockextra(true); - mp->schedlink = mnext; - unlockextra(mp); -} - -#define MLOCKED ((M*)1) - -// lockextra locks the extra list and returns the list head. -// The caller must unlock the list by storing a new list head -// to runtime.extram. If nilokay is true, then lockextra will -// return a nil list head if that's what it finds. If nilokay is false, -// lockextra will keep waiting until the list head is no longer nil. -#pragma textflag NOSPLIT -static M* -lockextra(bool nilokay) -{ - M *mp; - void (*yield)(void); - - for(;;) { - mp = runtime·atomicloadp(&runtime·extram); - if(mp == MLOCKED) { - yield = runtime·osyield; - yield(); - continue; - } - if(mp == nil && !nilokay) { - runtime·usleep(1); - continue; - } - if(!runtime·casp(&runtime·extram, mp, MLOCKED)) { - yield = runtime·osyield; - yield(); - continue; - } - break; - } - return mp; -} - -#pragma textflag NOSPLIT -static void -unlockextra(M *mp) -{ - runtime·atomicstorep(&runtime·extram, mp); -} - - -// Create a new m. It will start off with a call to fn, or else the scheduler. -static void -newm(void(*fn)(void), P *p) -{ - M *mp; - - mp = runtime·allocm(p); - mp->nextp = p; - mp->mstartfn = fn; - - if(runtime·iscgo) { - CgoThreadStart ts; - - if(_cgo_thread_start == nil) - runtime·throw("_cgo_thread_start missing"); - ts.g = mp->g0; - ts.tls = mp->tls; - ts.fn = runtime·mstart; - runtime·asmcgocall(_cgo_thread_start, &ts); - return; - } - runtime·newosproc(mp, (byte*)mp->g0->stack.hi); -} - -// Stops execution of the current m until new work is available. -// Returns with acquired P. -static void -stopm(void) -{ - if(g->m->locks) - runtime·throw("stopm holding locks"); - if(g->m->p) - runtime·throw("stopm holding p"); - if(g->m->spinning) { - g->m->spinning = false; - runtime·xadd(&runtime·sched.nmspinning, -1); - } - -retry: - runtime·lock(&runtime·sched.lock); - mput(g->m); - runtime·unlock(&runtime·sched.lock); - runtime·notesleep(&g->m->park); - runtime·noteclear(&g->m->park); - if(g->m->helpgc) { - runtime·gchelper(); - g->m->helpgc = 0; - g->m->mcache = nil; - goto retry; - } - acquirep(g->m->nextp); - g->m->nextp = nil; -} - -static void -mspinning(void) -{ - g->m->spinning = true; -} - -// Schedules some M to run the p (creates an M if necessary). -// If p==nil, tries to get an idle P, if no idle P's does nothing. -static void -startm(P *p, bool spinning) -{ - M *mp; - void (*fn)(void); - - runtime·lock(&runtime·sched.lock); - if(p == nil) { - p = pidleget(); - if(p == nil) { - runtime·unlock(&runtime·sched.lock); - if(spinning) - runtime·xadd(&runtime·sched.nmspinning, -1); - return; - } - } - mp = mget(); - runtime·unlock(&runtime·sched.lock); - if(mp == nil) { - fn = nil; - if(spinning) - fn = mspinning; - newm(fn, p); - return; - } - if(mp->spinning) - runtime·throw("startm: m is spinning"); - if(mp->nextp) - runtime·throw("startm: m has p"); - mp->spinning = spinning; - mp->nextp = p; - runtime·notewakeup(&mp->park); -} - -// Hands off P from syscall or locked M. -static void -handoffp(P *p) -{ - // if it has local work, start it straight away - if(p->runqhead != p->runqtail || runtime·sched.runqsize) { - startm(p, false); - return; - } - // no local work, check that there are no spinning/idle M's, - // otherwise our help is not required - if(runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) == 0 && // TODO: fast atomic - runtime·cas(&runtime·sched.nmspinning, 0, 1)){ - startm(p, true); - return; - } - runtime·lock(&runtime·sched.lock); - if(runtime·sched.gcwaiting) { - p->status = Pgcstop; - if(--runtime·sched.stopwait == 0) - runtime·notewakeup(&runtime·sched.stopnote); - runtime·unlock(&runtime·sched.lock); - return; - } - if(runtime·sched.runqsize) { - runtime·unlock(&runtime·sched.lock); - startm(p, false); - return; - } - // If this is the last running P and nobody is polling network, - // need to wakeup another M to poll network. - if(runtime·sched.npidle == runtime·gomaxprocs-1 && runtime·atomicload64(&runtime·sched.lastpoll) != 0) { - runtime·unlock(&runtime·sched.lock); - startm(p, false); - return; - } - pidleput(p); - runtime·unlock(&runtime·sched.lock); -} - -// Tries to add one more P to execute G's. -// Called when a G is made runnable (newproc, ready). -static void -wakep(void) -{ - // be conservative about spinning threads - if(!runtime·cas(&runtime·sched.nmspinning, 0, 1)) - return; - startm(nil, true); -} - -// Stops execution of the current m that is locked to a g until the g is runnable again. -// Returns with acquired P. -static void -stoplockedm(void) -{ - P *p; - uint32 status; - - if(g->m->lockedg == nil || g->m->lockedg->lockedm != g->m) - runtime·throw("stoplockedm: inconsistent locking"); - if(g->m->p) { - // Schedule another M to run this p. - p = releasep(); - handoffp(p); - } - incidlelocked(1); - // Wait until another thread schedules lockedg again. - runtime·notesleep(&g->m->park); - runtime·noteclear(&g->m->park); - status = runtime·readgstatus(g->m->lockedg); - if((status&~Gscan) != Grunnable){ - runtime·printf("runtime:stoplockedm: g is not Grunnable or Gscanrunnable"); - dumpgstatus(g); - runtime·throw("stoplockedm: not runnable"); - } - acquirep(g->m->nextp); - g->m->nextp = nil; -} - -// Schedules the locked m to run the locked gp. -static void -startlockedm(G *gp) -{ - M *mp; - P *p; - - mp = gp->lockedm; - if(mp == g->m) - runtime·throw("startlockedm: locked to me"); - if(mp->nextp) - runtime·throw("startlockedm: m has p"); - // directly handoff current P to the locked m - incidlelocked(-1); - p = releasep(); - mp->nextp = p; - runtime·notewakeup(&mp->park); - stopm(); -} - -// Stops the current m for stoptheworld. -// Returns when the world is restarted. -static void -gcstopm(void) -{ - P *p; - - if(!runtime·sched.gcwaiting) - runtime·throw("gcstopm: not waiting for gc"); - if(g->m->spinning) { - g->m->spinning = false; - runtime·xadd(&runtime·sched.nmspinning, -1); - } - p = releasep(); - runtime·lock(&runtime·sched.lock); - p->status = Pgcstop; - if(--runtime·sched.stopwait == 0) - runtime·notewakeup(&runtime·sched.stopnote); - runtime·unlock(&runtime·sched.lock); - stopm(); -} - -// Schedules gp to run on the current M. -// Never returns. -static void -execute(G *gp) -{ - int32 hz; - - runtime·casgstatus(gp, Grunnable, Grunning); - gp->waitsince = 0; - gp->preempt = false; - gp->stackguard0 = gp->stack.lo + StackGuard; - g->m->p->schedtick++; - g->m->curg = gp; - gp->m = g->m; - - // Check whether the profiler needs to be turned on or off. - hz = runtime·sched.profilehz; - if(g->m->profilehz != hz) - runtime·resetcpuprofiler(hz); - - runtime·gogo(&gp->sched); -} - -// Finds a runnable goroutine to execute. -// Tries to steal from other P's, get g from global queue, poll network. -static G* -findrunnable(void) -{ - G *gp; - P *p; - int32 i; - -top: - if(runtime·sched.gcwaiting) { - gcstopm(); - goto top; - } - if(runtime·fingwait && runtime·fingwake && (gp = runtime·wakefing()) != nil) - runtime·ready(gp); - // local runq - gp = runqget(g->m->p); - if(gp) - return gp; - // global runq - if(runtime·sched.runqsize) { - runtime·lock(&runtime·sched.lock); - gp = globrunqget(g->m->p, 0); - runtime·unlock(&runtime·sched.lock); - if(gp) - return gp; - } - // poll network - gp = runtime·netpoll(false); // non-blocking - if(gp) { - injectglist(gp->schedlink); - runtime·casgstatus(gp, Gwaiting, Grunnable); - return gp; - } - // If number of spinning M's >= number of busy P's, block. - // This is necessary to prevent excessive CPU consumption - // when GOMAXPROCS>>1 but the program parallelism is low. - if(!g->m->spinning && 2 * runtime·atomicload(&runtime·sched.nmspinning) >= runtime·gomaxprocs - runtime·atomicload(&runtime·sched.npidle)) // TODO: fast atomic - goto stop; - if(!g->m->spinning) { - g->m->spinning = true; - runtime·xadd(&runtime·sched.nmspinning, 1); - } - // random steal from other P's - for(i = 0; i < 2*runtime·gomaxprocs; i++) { - if(runtime·sched.gcwaiting) - goto top; - p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; - if(p == g->m->p) - gp = runqget(p); - else - gp = runqsteal(g->m->p, p); - if(gp) - return gp; - } -stop: - // return P and block - runtime·lock(&runtime·sched.lock); - if(runtime·sched.gcwaiting) { - runtime·unlock(&runtime·sched.lock); - goto top; - } - if(runtime·sched.runqsize) { - gp = globrunqget(g->m->p, 0); - runtime·unlock(&runtime·sched.lock); - return gp; - } - p = releasep(); - pidleput(p); - runtime·unlock(&runtime·sched.lock); - if(g->m->spinning) { - g->m->spinning = false; - runtime·xadd(&runtime·sched.nmspinning, -1); - } - // check all runqueues once again - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - if(p && p->runqhead != p->runqtail) { - runtime·lock(&runtime·sched.lock); - p = pidleget(); - runtime·unlock(&runtime·sched.lock); - if(p) { - acquirep(p); - goto top; - } - break; - } - } - // poll network - if(runtime·xchg64(&runtime·sched.lastpoll, 0) != 0) { - if(g->m->p) - runtime·throw("findrunnable: netpoll with p"); - if(g->m->spinning) - runtime·throw("findrunnable: netpoll with spinning"); - gp = runtime·netpoll(true); // block until new work is available - runtime·atomicstore64(&runtime·sched.lastpoll, runtime·nanotime()); - if(gp) { - runtime·lock(&runtime·sched.lock); - p = pidleget(); - runtime·unlock(&runtime·sched.lock); - if(p) { - acquirep(p); - injectglist(gp->schedlink); - runtime·casgstatus(gp, Gwaiting, Grunnable); - return gp; - } - injectglist(gp); - } - } - stopm(); - goto top; -} - -static void -resetspinning(void) -{ - int32 nmspinning; - - if(g->m->spinning) { - g->m->spinning = false; - nmspinning = runtime·xadd(&runtime·sched.nmspinning, -1); - if(nmspinning < 0) - runtime·throw("findrunnable: negative nmspinning"); - } else - nmspinning = runtime·atomicload(&runtime·sched.nmspinning); - - // M wakeup policy is deliberately somewhat conservative (see nmspinning handling), - // so see if we need to wakeup another P here. - if (nmspinning == 0 && runtime·atomicload(&runtime·sched.npidle) > 0) - wakep(); -} - -// Injects the list of runnable G's into the scheduler. -// Can run concurrently with GC. -static void -injectglist(G *glist) -{ - int32 n; - G *gp; - - if(glist == nil) - return; - runtime·lock(&runtime·sched.lock); - for(n = 0; glist; n++) { - gp = glist; - glist = gp->schedlink; - runtime·casgstatus(gp, Gwaiting, Grunnable); - globrunqput(gp); - } - runtime·unlock(&runtime·sched.lock); - - for(; n && runtime·sched.npidle; n--) - startm(nil, false); -} - -// One round of scheduler: find a runnable goroutine and execute it. -// Never returns. -static void -schedule(void) -{ - G *gp; - uint32 tick; - - if(g->m->locks) - runtime·throw("schedule: holding locks"); - - if(g->m->lockedg) { - stoplockedm(); - execute(g->m->lockedg); // Never returns. - } - -top: - if(runtime·sched.gcwaiting) { - gcstopm(); - goto top; - } - - gp = nil; - // Check the global runnable queue once in a while to ensure fairness. - // Otherwise two goroutines can completely occupy the local runqueue - // by constantly respawning each other. - tick = g->m->p->schedtick; - // This is a fancy way to say tick%61==0, - // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors. - if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runqsize > 0) { - runtime·lock(&runtime·sched.lock); - gp = globrunqget(g->m->p, 1); - runtime·unlock(&runtime·sched.lock); - if(gp) - resetspinning(); - } - if(gp == nil) { - gp = runqget(g->m->p); - if(gp && g->m->spinning) - runtime·throw("schedule: spinning with local work"); - } - if(gp == nil) { - gp = findrunnable(); // blocks until work is available - resetspinning(); - } - - if(gp->lockedm) { - // Hands off own p to the locked m, - // then blocks waiting for a new p. - startlockedm(gp); - goto top; - } - - execute(gp); -} - -// dropg removes the association between m and the current goroutine m->curg (gp for short). -// Typically a caller sets gp's status away from Grunning and then -// immediately calls dropg to finish the job. The caller is also responsible -// for arranging that gp will be restarted using runtime·ready at an -// appropriate time. After calling dropg and arranging for gp to be -// readied later, the caller can do other work but eventually should -// call schedule to restart the scheduling of goroutines on this m. -static void -dropg(void) -{ - if(g->m->lockedg == nil) { - g->m->curg->m = nil; - g->m->curg = nil; - } -} - -// Puts the current goroutine into a waiting state and calls unlockf. -// If unlockf returns false, the goroutine is resumed. -void -runtime·park(bool(*unlockf)(G*, void*), void *lock, String reason) -{ - void (*fn)(G*); - - g->m->waitlock = lock; - g->m->waitunlockf = unlockf; - g->waitreason = reason; - fn = runtime·park_m; - runtime·mcall(&fn); -} - -bool -runtime·parkunlock_c(G *gp, void *lock) -{ - USED(gp); - runtime·unlock(lock); - return true; -} - -// Puts the current goroutine into a waiting state and unlocks the lock. -// The goroutine can be made runnable again by calling runtime·ready(gp). -void -runtime·parkunlock(Mutex *lock, String reason) -{ - runtime·park(runtime·parkunlock_c, lock, reason); -} - -// runtime·park continuation on g0. -void -runtime·park_m(G *gp) -{ - bool ok; - - runtime·casgstatus(gp, Grunning, Gwaiting); - dropg(); - - if(g->m->waitunlockf) { - ok = g->m->waitunlockf(gp, g->m->waitlock); - g->m->waitunlockf = nil; - g->m->waitlock = nil; - if(!ok) { - runtime·casgstatus(gp, Gwaiting, Grunnable); - execute(gp); // Schedule it back, never returns. - } - } - - schedule(); -} - -// Gosched continuation on g0. -void -runtime·gosched_m(G *gp) -{ - uint32 status; - - status = runtime·readgstatus(gp); - if((status&~Gscan) != Grunning){ - dumpgstatus(gp); - runtime·throw("bad g status"); - } - runtime·casgstatus(gp, Grunning, Grunnable); - dropg(); - runtime·lock(&runtime·sched.lock); - globrunqput(gp); - runtime·unlock(&runtime·sched.lock); - - schedule(); -} - -// Finishes execution of the current goroutine. -// Must be NOSPLIT because it is called from Go. -#pragma textflag NOSPLIT -void -runtime·goexit1(void) -{ - void (*fn)(G*); - - if(raceenabled) - runtime·racegoend(); - fn = goexit0; - runtime·mcall(&fn); -} - -// runtime·goexit continuation on g0. -static void -goexit0(G *gp) -{ - runtime·casgstatus(gp, Grunning, Gdead); - gp->m = nil; - gp->lockedm = nil; - g->m->lockedg = nil; - gp->paniconfault = 0; - gp->defer = nil; // should be true already but just in case. - gp->panic = nil; // non-nil for Goexit during panic. points at stack-allocated data. - gp->writebuf.array = nil; - gp->writebuf.len = 0; - gp->writebuf.cap = 0; - gp->waitreason.str = nil; - gp->waitreason.len = 0; - gp->param = nil; - - dropg(); - - if(g->m->locked & ~LockExternal) { - runtime·printf("invalid m->locked = %d\n", g->m->locked); - runtime·throw("internal lockOSThread error"); - } - g->m->locked = 0; - gfput(g->m->p, gp); - schedule(); -} - -#pragma textflag NOSPLIT -static void -save(uintptr pc, uintptr sp) -{ - g->sched.pc = pc; - g->sched.sp = sp; - g->sched.lr = 0; - g->sched.ret = 0; - g->sched.ctxt = 0; - g->sched.g = g; -} - -static void entersyscall_bad(void); -static void entersyscall_sysmon(void); -static void entersyscall_gcwait(void); - -// The goroutine g is about to enter a system call. -// Record that it's not using the cpu anymore. -// This is called only from the go syscall library and cgocall, -// not from the low-level system calls used by the runtime. -// -// Entersyscall cannot split the stack: the runtime·gosave must -// make g->sched refer to the caller's stack segment, because -// entersyscall is going to return immediately after. -// -// Nothing entersyscall calls can split the stack either. -// We cannot safely move the stack during an active call to syscall, -// because we do not know which of the uintptr arguments are -// really pointers (back into the stack). -// In practice, this means that we make the fast path run through -// entersyscall doing no-split things, and the slow path has to use onM -// to run bigger things on the m stack. -// -// reentersyscall is the entry point used by cgo callbacks, where explicitly -// saved SP and PC are restored. This is needed when exitsyscall will be called -// from a function further up in the call stack than the parent, as g->syscallsp -// must always point to a valid stack frame. entersyscall below is the normal -// entry point for syscalls, which obtains the SP and PC from the caller. -#pragma textflag NOSPLIT -void -runtime·reentersyscall(uintptr pc, uintptr sp) -{ - void (*fn)(void); - - // Disable preemption because during this function g is in Gsyscall status, - // but can have inconsistent g->sched, do not let GC observe it. - g->m->locks++; - - // Entersyscall must not call any function that might split/grow the stack. - // (See details in comment above.) - // Catch calls that might, by replacing the stack guard with something that - // will trip any stack check and leaving a flag to tell newstack to die. - g->stackguard0 = StackPreempt; - g->throwsplit = 1; - - // Leave SP around for GC and traceback. - save(pc, sp); - g->syscallsp = sp; - g->syscallpc = pc; - runtime·casgstatus(g, Grunning, Gsyscall); - if(g->syscallsp < g->stack.lo || g->stack.hi < g->syscallsp) { - fn = entersyscall_bad; - runtime·onM(&fn); - } - - if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic - fn = entersyscall_sysmon; - runtime·onM(&fn); - save(pc, sp); - } - - g->m->mcache = nil; - g->m->p->m = nil; - runtime·atomicstore(&g->m->p->status, Psyscall); - if(runtime·sched.gcwaiting) { - fn = entersyscall_gcwait; - runtime·onM(&fn); - save(pc, sp); - } - - // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). - // We set stackguard to StackPreempt so that first split stack check calls morestack. - // Morestack detects this case and throws. - g->stackguard0 = StackPreempt; - g->m->locks--; -} - -// Standard syscall entry used by the go syscall library and normal cgo calls. -#pragma textflag NOSPLIT -void -·entersyscall(int32 dummy) -{ - runtime·reentersyscall((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); -} - -static void -entersyscall_bad(void) -{ - G *gp; - - gp = g->m->curg; - runtime·printf("entersyscall inconsistent %p [%p,%p]\n", - gp->syscallsp, gp->stack.lo, gp->stack.hi); - runtime·throw("entersyscall"); -} - -static void -entersyscall_sysmon(void) -{ - runtime·lock(&runtime·sched.lock); - if(runtime·atomicload(&runtime·sched.sysmonwait)) { - runtime·atomicstore(&runtime·sched.sysmonwait, 0); - runtime·notewakeup(&runtime·sched.sysmonnote); - } - runtime·unlock(&runtime·sched.lock); -} - -static void -entersyscall_gcwait(void) -{ - runtime·lock(&runtime·sched.lock); - if (runtime·sched.stopwait > 0 && runtime·cas(&g->m->p->status, Psyscall, Pgcstop)) { - if(--runtime·sched.stopwait == 0) - runtime·notewakeup(&runtime·sched.stopnote); - } - runtime·unlock(&runtime·sched.lock); -} - -static void entersyscallblock_handoff(void); - -// The same as runtime·entersyscall(), but with a hint that the syscall is blocking. -#pragma textflag NOSPLIT -void -·entersyscallblock(int32 dummy) -{ - void (*fn)(void); - - g->m->locks++; // see comment in entersyscall - g->throwsplit = 1; - g->stackguard0 = StackPreempt; // see comment in entersyscall - - // Leave SP around for GC and traceback. - save((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); - g->syscallsp = g->sched.sp; - g->syscallpc = g->sched.pc; - runtime·casgstatus(g, Grunning, Gsyscall); - if(g->syscallsp < g->stack.lo || g->stack.hi < g->syscallsp) { - fn = entersyscall_bad; - runtime·onM(&fn); - } - - fn = entersyscallblock_handoff; - runtime·onM(&fn); - - // Resave for traceback during blocked call. - save((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); - - g->m->locks--; -} - -static void -entersyscallblock_handoff(void) -{ - handoffp(releasep()); -} - -// The goroutine g exited its system call. -// Arrange for it to run on a cpu again. -// This is called only from the go syscall library, not -// from the low-level system calls used by the runtime. -#pragma textflag NOSPLIT -void -·exitsyscall(int32 dummy) -{ - void (*fn)(G*); - - g->m->locks++; // see comment in entersyscall - - if(runtime·getcallersp(&dummy) > g->syscallsp) - runtime·throw("exitsyscall: syscall frame is no longer valid"); - - g->waitsince = 0; - if(exitsyscallfast()) { - // There's a cpu for us, so we can run. - g->m->p->syscalltick++; - // We need to cas the status and scan before resuming... - runtime·casgstatus(g, Gsyscall, Grunning); - - // Garbage collector isn't running (since we are), - // so okay to clear syscallsp. - g->syscallsp = (uintptr)nil; - g->m->locks--; - if(g->preempt) { - // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - } else { - // otherwise restore the real stackguard, we've spoiled it in entersyscall/entersyscallblock - g->stackguard0 = g->stack.lo + StackGuard; - } - g->throwsplit = 0; - return; - } - - g->m->locks--; - - // Call the scheduler. - fn = exitsyscall0; - runtime·mcall(&fn); - - // Scheduler returned, so we're allowed to run now. - // Delete the syscallsp information that we left for - // the garbage collector during the system call. - // Must wait until now because until gosched returns - // we don't know for sure that the garbage collector - // is not running. - g->syscallsp = (uintptr)nil; - g->m->p->syscalltick++; - g->throwsplit = 0; -} - -static void exitsyscallfast_pidle(void); - -#pragma textflag NOSPLIT -static bool -exitsyscallfast(void) -{ - void (*fn)(void); - - // Freezetheworld sets stopwait but does not retake P's. - if(runtime·sched.stopwait) { - g->m->p = nil; - return false; - } - - // Try to re-acquire the last P. - if(g->m->p && g->m->p->status == Psyscall && runtime·cas(&g->m->p->status, Psyscall, Prunning)) { - // There's a cpu for us, so we can run. - g->m->mcache = g->m->p->mcache; - g->m->p->m = g->m; - return true; - } - // Try to get any other idle P. - g->m->p = nil; - if(runtime·sched.pidle) { - fn = exitsyscallfast_pidle; - runtime·onM(&fn); - if(g->m->scalararg[0]) { - g->m->scalararg[0] = 0; - return true; - } - } - return false; -} - -static void -exitsyscallfast_pidle(void) -{ - P *p; - - runtime·lock(&runtime·sched.lock); - p = pidleget(); - if(p && runtime·atomicload(&runtime·sched.sysmonwait)) { - runtime·atomicstore(&runtime·sched.sysmonwait, 0); - runtime·notewakeup(&runtime·sched.sysmonnote); - } - runtime·unlock(&runtime·sched.lock); - if(p) { - acquirep(p); - g->m->scalararg[0] = 1; - } else - g->m->scalararg[0] = 0; -} - -// runtime·exitsyscall slow path on g0. -// Failed to acquire P, enqueue gp as runnable. -static void -exitsyscall0(G *gp) -{ - P *p; - - runtime·casgstatus(gp, Gsyscall, Grunnable); - dropg(); - runtime·lock(&runtime·sched.lock); - p = pidleget(); - if(p == nil) - globrunqput(gp); - else if(runtime·atomicload(&runtime·sched.sysmonwait)) { - runtime·atomicstore(&runtime·sched.sysmonwait, 0); - runtime·notewakeup(&runtime·sched.sysmonnote); - } - runtime·unlock(&runtime·sched.lock); - if(p) { - acquirep(p); - execute(gp); // Never returns. - } - if(g->m->lockedg) { - // Wait until another thread schedules gp and so m again. - stoplockedm(); - execute(gp); // Never returns. - } - stopm(); - schedule(); // Never returns. -} - -static void -beforefork(void) -{ - G *gp; - - gp = g->m->curg; - // Fork can hang if preempted with signals frequently enough (see issue 5517). - // Ensure that we stay on the same M where we disable profiling. - gp->m->locks++; - if(gp->m->profilehz != 0) - runtime·resetcpuprofiler(0); - - // This function is called before fork in syscall package. - // Code between fork and exec must not allocate memory nor even try to grow stack. - // Here we spoil g->stackguard to reliably detect any attempts to grow stack. - // runtime_AfterFork will undo this in parent process, but not in child. - gp->stackguard0 = StackFork; -} - -// Called from syscall package before fork. -#pragma textflag NOSPLIT -void -syscall·runtime_BeforeFork(void) -{ - void (*fn)(void); - - fn = beforefork; - runtime·onM(&fn); -} - -static void -afterfork(void) -{ - int32 hz; - G *gp; - - gp = g->m->curg; - // See the comment in runtime_BeforeFork. - gp->stackguard0 = gp->stack.lo + StackGuard; - - hz = runtime·sched.profilehz; - if(hz != 0) - runtime·resetcpuprofiler(hz); - gp->m->locks--; -} - -// Called from syscall package after fork in parent. -#pragma textflag NOSPLIT -void -syscall·runtime_AfterFork(void) -{ - void (*fn)(void); - - fn = afterfork; - runtime·onM(&fn); -} - -// Hook used by runtime·malg to call runtime·stackalloc on the -// scheduler stack. This exists because runtime·stackalloc insists -// on being called on the scheduler stack, to avoid trying to grow -// the stack while allocating a new stack segment. -static void -mstackalloc(G *gp) -{ - G *newg; - uintptr size; - - newg = g->m->ptrarg[0]; - size = g->m->scalararg[0]; - - newg->stack = runtime·stackalloc(size); - - runtime·gogo(&gp->sched); -} - -// Allocate a new g, with a stack big enough for stacksize bytes. -G* -runtime·malg(int32 stacksize) -{ - G *newg; - void (*fn)(G*); - - newg = allocg(); - if(stacksize >= 0) { - stacksize = runtime·round2(StackSystem + stacksize); - if(g == g->m->g0) { - // running on scheduler stack already. - newg->stack = runtime·stackalloc(stacksize); - } else { - // have to call stackalloc on scheduler stack. - g->m->scalararg[0] = stacksize; - g->m->ptrarg[0] = newg; - fn = mstackalloc; - runtime·mcall(&fn); - g->m->ptrarg[0] = nil; - } - newg->stackguard0 = newg->stack.lo + StackGuard; - newg->stackguard1 = ~(uintptr)0; - } - return newg; -} - -static void -newproc_m(void) -{ - byte *argp; - void *callerpc; - FuncVal *fn; - int32 siz; - - siz = g->m->scalararg[0]; - callerpc = (void*)g->m->scalararg[1]; - argp = g->m->ptrarg[0]; - fn = (FuncVal*)g->m->ptrarg[1]; - - runtime·newproc1(fn, argp, siz, 0, callerpc); - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; -} - -// Create a new g running fn with siz bytes of arguments. -// Put it on the queue of g's waiting to run. -// The compiler turns a go statement into a call to this. -// Cannot split the stack because it assumes that the arguments -// are available sequentially after &fn; they would not be -// copied if a stack split occurred. -#pragma textflag NOSPLIT -void -runtime·newproc(int32 siz, FuncVal* fn, ...) -{ - byte *argp; - void (*mfn)(void); - - if(thechar == '5') - argp = (byte*)(&fn+2); // skip caller's saved LR - else - argp = (byte*)(&fn+1); - - g->m->locks++; - g->m->scalararg[0] = siz; - g->m->scalararg[1] = (uintptr)runtime·getcallerpc(&siz); - g->m->ptrarg[0] = argp; - g->m->ptrarg[1] = fn; - mfn = newproc_m; - runtime·onM(&mfn); - g->m->locks--; -} - -void runtime·main(void); - -// Create a new g running fn with narg bytes of arguments starting -// at argp and returning nret bytes of results. callerpc is the -// address of the go statement that created this. The new g is put -// on the queue of g's waiting to run. -G* -runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc) -{ - byte *sp; - G *newg; - P *p; - int32 siz; - - if(fn == nil) { - g->m->throwing = -1; // do not dump full stacks - runtime·throw("go of nil func value"); - } - g->m->locks++; // disable preemption because it can be holding p in a local var - siz = narg + nret; - siz = (siz+7) & ~7; - - // We could allocate a larger initial stack if necessary. - // Not worth it: this is almost always an error. - // 4*sizeof(uintreg): extra space added below - // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). - if(siz >= StackMin - 4*sizeof(uintreg) - sizeof(uintreg)) - runtime·throw("runtime.newproc: function arguments too large for new goroutine"); - - p = g->m->p; - if((newg = gfget(p)) == nil) { - newg = runtime·malg(StackMin); - runtime·casgstatus(newg, Gidle, Gdead); - runtime·allgadd(newg); // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. - } - if(newg->stack.hi == 0) - runtime·throw("newproc1: newg missing stack"); - - if(runtime·readgstatus(newg) != Gdead) - runtime·throw("newproc1: new g is not Gdead"); - - sp = (byte*)newg->stack.hi; - sp -= 4*sizeof(uintreg); // extra space in case of reads slightly beyond frame - sp -= siz; - runtime·memmove(sp, argp, narg); - if(thechar == '5') { - // caller's LR - sp -= sizeof(void*); - *(void**)sp = nil; - } - - runtime·memclr((byte*)&newg->sched, sizeof newg->sched); - newg->sched.sp = (uintptr)sp; - newg->sched.pc = (uintptr)runtime·goexit + PCQuantum; // +PCQuantum so that previous instruction is in same function - newg->sched.g = newg; - runtime·gostartcallfn(&newg->sched, fn); - newg->gopc = (uintptr)callerpc; - runtime·casgstatus(newg, Gdead, Grunnable); - - if(p->goidcache == p->goidcacheend) { - // Sched.goidgen is the last allocated id, - // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. - // At startup sched.goidgen=0, so main goroutine receives goid=1. - p->goidcache = runtime·xadd64(&runtime·sched.goidgen, GoidCacheBatch); - p->goidcache -= GoidCacheBatch - 1; - p->goidcacheend = p->goidcache + GoidCacheBatch; - } - newg->goid = p->goidcache++; - if(raceenabled) - newg->racectx = runtime·racegostart((void*)callerpc); - runqput(p, newg); - - if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0 && fn->fn != runtime·main) // TODO: fast atomic - wakep(); - g->m->locks--; - if(g->m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - return newg; -} - -// Put on gfree list. -// If local list is too long, transfer a batch to the global list. -static void -gfput(P *p, G *gp) -{ - uintptr stksize; - - if(runtime·readgstatus(gp) != Gdead) - runtime·throw("gfput: bad status (not Gdead)"); - - stksize = gp->stack.hi - gp->stack.lo; - - if(stksize != FixedStack) { - // non-standard stack size - free it. - runtime·stackfree(gp->stack); - gp->stack.lo = 0; - gp->stack.hi = 0; - gp->stackguard0 = 0; - } - gp->schedlink = p->gfree; - p->gfree = gp; - p->gfreecnt++; - if(p->gfreecnt >= 64) { - runtime·lock(&runtime·sched.gflock); - while(p->gfreecnt >= 32) { - p->gfreecnt--; - gp = p->gfree; - p->gfree = gp->schedlink; - gp->schedlink = runtime·sched.gfree; - runtime·sched.gfree = gp; - runtime·sched.ngfree++; - } - runtime·unlock(&runtime·sched.gflock); - } -} - -// Get from gfree list. -// If local list is empty, grab a batch from global list. -static G* -gfget(P *p) -{ - G *gp; - void (*fn)(G*); - -retry: - gp = p->gfree; - if(gp == nil && runtime·sched.gfree) { - runtime·lock(&runtime·sched.gflock); - while(p->gfreecnt < 32 && runtime·sched.gfree != nil) { - p->gfreecnt++; - gp = runtime·sched.gfree; - runtime·sched.gfree = gp->schedlink; - runtime·sched.ngfree--; - gp->schedlink = p->gfree; - p->gfree = gp; - } - runtime·unlock(&runtime·sched.gflock); - goto retry; - } - if(gp) { - p->gfree = gp->schedlink; - p->gfreecnt--; - - if(gp->stack.lo == 0) { - // Stack was deallocated in gfput. Allocate a new one. - if(g == g->m->g0) { - gp->stack = runtime·stackalloc(FixedStack); - } else { - g->m->scalararg[0] = FixedStack; - g->m->ptrarg[0] = gp; - fn = mstackalloc; - runtime·mcall(&fn); - g->m->ptrarg[0] = nil; - } - gp->stackguard0 = gp->stack.lo + StackGuard; - } else { - if(raceenabled) - runtime·racemalloc((void*)gp->stack.lo, gp->stack.hi - gp->stack.lo); - } - } - return gp; -} - -// Purge all cached G's from gfree list to the global list. -static void -gfpurge(P *p) -{ - G *gp; - - runtime·lock(&runtime·sched.gflock); - while(p->gfreecnt != 0) { - p->gfreecnt--; - gp = p->gfree; - p->gfree = gp->schedlink; - gp->schedlink = runtime·sched.gfree; - runtime·sched.gfree = gp; - runtime·sched.ngfree++; - } - runtime·unlock(&runtime·sched.gflock); -} - -#pragma textflag NOSPLIT -void -runtime·Breakpoint(void) -{ - runtime·breakpoint(); -} - -// lockOSThread is called by runtime.LockOSThread and runtime.lockOSThread below -// after they modify m->locked. Do not allow preemption during this call, -// or else the m might be different in this function than in the caller. -#pragma textflag NOSPLIT -static void -lockOSThread(void) -{ - g->m->lockedg = g; - g->lockedm = g->m; -} - -#pragma textflag NOSPLIT -void -runtime·LockOSThread(void) -{ - g->m->locked |= LockExternal; - lockOSThread(); -} - -#pragma textflag NOSPLIT -void -runtime·lockOSThread(void) -{ - g->m->locked += LockInternal; - lockOSThread(); -} - - -// unlockOSThread is called by runtime.UnlockOSThread and runtime.unlockOSThread below -// after they update m->locked. Do not allow preemption during this call, -// or else the m might be in different in this function than in the caller. -#pragma textflag NOSPLIT -static void -unlockOSThread(void) -{ - if(g->m->locked != 0) - return; - g->m->lockedg = nil; - g->lockedm = nil; -} - -#pragma textflag NOSPLIT -void -runtime·UnlockOSThread(void) -{ - g->m->locked &= ~LockExternal; - unlockOSThread(); -} - -static void badunlockOSThread(void); - -#pragma textflag NOSPLIT -void -runtime·unlockOSThread(void) -{ - void (*fn)(void); - - if(g->m->locked < LockInternal) { - fn = badunlockOSThread; - runtime·onM(&fn); - } - g->m->locked -= LockInternal; - unlockOSThread(); -} - -static void -badunlockOSThread(void) -{ - runtime·throw("runtime: internal error: misuse of lockOSThread/unlockOSThread"); -} - -#pragma textflag NOSPLIT -int32 -runtime·gcount(void) -{ - P *p, **pp; - int32 n; - - n = runtime·allglen - runtime·sched.ngfree; - for(pp=runtime·allp; p=*pp; pp++) - n -= p->gfreecnt; - // All these variables can be changed concurrently, so the result can be inconsistent. - // But at least the current goroutine is running. - if(n < 1) - n = 1; - return n; -} - -int32 -runtime·mcount(void) -{ - return runtime·sched.mcount; -} - -static struct ProfState { - uint32 lock; - int32 hz; -} prof; - -static void System(void) { System(); } -static void ExternalCode(void) { ExternalCode(); } -static void GC(void) { GC(); } - -extern void runtime·cpuproftick(uintptr*, int32); -extern byte runtime·etext[]; - -// Called if we receive a SIGPROF signal. -void -runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) -{ - int32 n; - bool traceback; - // Do not use global m in this function, use mp instead. - // On windows one m is sending reports about all the g's, so m means a wrong thing. - byte m; - uintptr stk[100]; - - m = 0; - USED(m); - - if(prof.hz == 0) - return; - - // Profiling runs concurrently with GC, so it must not allocate. - mp->mallocing++; - - // Define that a "user g" is a user-created goroutine, and a "system g" - // is one that is m->g0 or m->gsignal. We've only made sure that we - // can unwind user g's, so exclude the system g's. - // - // It is not quite as easy as testing gp == m->curg (the current user g) - // because we might be interrupted for profiling halfway through a - // goroutine switch. The switch involves updating three (or four) values: - // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, - // because once it gets updated the new g is running. - // - // When switching from a user g to a system g, LR is not considered live, - // so the update only affects g, SP, and PC. Since PC must be last, there - // the possible partial transitions in ordinary execution are (1) g alone is updated, - // (2) both g and SP are updated, and (3) SP alone is updated. - // If g is updated, we'll see a system g and not look closer. - // If SP alone is updated, we can detect the partial transition by checking - // whether the SP is within g's stack bounds. (We could also require that SP - // be changed only after g, but the stack bounds check is needed by other - // cases, so there is no need to impose an additional requirement.) - // - // There is one exceptional transition to a system g, not in ordinary execution. - // When a signal arrives, the operating system starts the signal handler running - // with an updated PC and SP. The g is updated last, at the beginning of the - // handler. There are two reasons this is okay. First, until g is updated the - // g and SP do not match, so the stack bounds check detects the partial transition. - // Second, signal handlers currently run with signals disabled, so a profiling - // signal cannot arrive during the handler. - // - // When switching from a system g to a user g, there are three possibilities. - // - // First, it may be that the g switch has no PC update, because the SP - // either corresponds to a user g throughout (as in runtime.asmcgocall) - // or because it has been arranged to look like a user g frame - // (as in runtime.cgocallback_gofunc). In this case, since the entire - // transition is a g+SP update, a partial transition updating just one of - // those will be detected by the stack bounds check. - // - // Second, when returning from a signal handler, the PC and SP updates - // are performed by the operating system in an atomic update, so the g - // update must be done before them. The stack bounds check detects - // the partial transition here, and (again) signal handlers run with signals - // disabled, so a profiling signal cannot arrive then anyway. - // - // Third, the common case: it may be that the switch updates g, SP, and PC - // separately, as in runtime.gogo. - // - // Because runtime.gogo is the only instance, we check whether the PC lies - // within that function, and if so, not ask for a traceback. This approach - // requires knowing the size of the runtime.gogo function, which we - // record in arch_*.h and check in runtime_test.go. - // - // There is another apparently viable approach, recorded here in case - // the "PC within runtime.gogo" check turns out not to be usable. - // It would be possible to delay the update of either g or SP until immediately - // before the PC update instruction. Then, because of the stack bounds check, - // the only problematic interrupt point is just before that PC update instruction, - // and the sigprof handler can detect that instruction and simulate stepping past - // it in order to reach a consistent state. On ARM, the update of g must be made - // in two places (in R10 and also in a TLS slot), so the delayed update would - // need to be the SP update. The sigprof handler must read the instruction at - // the current PC and if it was the known instruction (for example, JMP BX or - // MOV R2, PC), use that other register in place of the PC value. - // The biggest drawback to this solution is that it requires that we can tell - // whether it's safe to read from the memory pointed at by PC. - // In a correct program, we can test PC == nil and otherwise read, - // but if a profiling signal happens at the instant that a program executes - // a bad jump (before the program manages to handle the resulting fault) - // the profiling handler could fault trying to read nonexistent memory. - // - // To recap, there are no constraints on the assembly being used for the - // transition. We simply require that g and SP match and that the PC is not - // in runtime.gogo. - traceback = true; - if(gp == nil || gp != mp->curg || - (uintptr)sp < gp->stack.lo || gp->stack.hi < (uintptr)sp || - ((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGogoBytes)) - traceback = false; - - n = 0; - if(traceback) - n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, stk, nelem(stk), nil, nil, TraceTrap); - if(!traceback || n <= 0) { - // Normal traceback is impossible or has failed. - // See if it falls into several common cases. - n = 0; - if(mp->ncgo > 0 && mp->curg != nil && - mp->curg->syscallpc != 0 && mp->curg->syscallsp != 0) { - // Cgo, we can't unwind and symbolize arbitrary C code, - // so instead collect Go stack that leads to the cgo call. - // This is especially important on windows, since all syscalls are cgo calls. - n = runtime·gentraceback(mp->curg->syscallpc, mp->curg->syscallsp, 0, mp->curg, 0, stk, nelem(stk), nil, nil, 0); - } -#ifdef GOOS_windows - if(n == 0 && mp->libcallg != nil && mp->libcallpc != 0 && mp->libcallsp != 0) { - // Libcall, i.e. runtime syscall on windows. - // Collect Go stack that leads to the call. - n = runtime·gentraceback(mp->libcallpc, mp->libcallsp, 0, mp->libcallg, 0, stk, nelem(stk), nil, nil, 0); - } -#endif - if(n == 0) { - // If all of the above has failed, account it against abstract "System" or "GC". - n = 2; - // "ExternalCode" is better than "etext". - if((uintptr)pc > (uintptr)runtime·etext) - pc = (byte*)ExternalCode + PCQuantum; - stk[0] = (uintptr)pc; - if(mp->gcing || mp->helpgc) - stk[1] = (uintptr)GC + PCQuantum; - else - stk[1] = (uintptr)System + PCQuantum; - } - } - - if(prof.hz != 0) { - // Simple cas-lock to coordinate with setcpuprofilerate. - while(!runtime·cas(&prof.lock, 0, 1)) - runtime·osyield(); - if(prof.hz != 0) - runtime·cpuproftick(stk, n); - runtime·atomicstore(&prof.lock, 0); - } - mp->mallocing--; -} - -// Arrange to call fn with a traceback hz times a second. -void -runtime·setcpuprofilerate_m(void) -{ - int32 hz; - - hz = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - - // Force sane arguments. - if(hz < 0) - hz = 0; - - // Disable preemption, otherwise we can be rescheduled to another thread - // that has profiling enabled. - g->m->locks++; - - // Stop profiler on this thread so that it is safe to lock prof. - // if a profiling signal came in while we had prof locked, - // it would deadlock. - runtime·resetcpuprofiler(0); - - while(!runtime·cas(&prof.lock, 0, 1)) - runtime·osyield(); - prof.hz = hz; - runtime·atomicstore(&prof.lock, 0); - - runtime·lock(&runtime·sched.lock); - runtime·sched.profilehz = hz; - runtime·unlock(&runtime·sched.lock); - - if(hz != 0) - runtime·resetcpuprofiler(hz); - - g->m->locks--; -} - -P *runtime·newP(void); - -// Change number of processors. The world is stopped, sched is locked. -static void -procresize(int32 new) -{ - int32 i, old; - bool empty; - G *gp; - P *p; - - old = runtime·gomaxprocs; - if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) - runtime·throw("procresize: invalid arg"); - // initialize new P's - for(i = 0; i < new; i++) { - p = runtime·allp[i]; - if(p == nil) { - p = runtime·newP(); - p->id = i; - p->status = Pgcstop; - runtime·atomicstorep(&runtime·allp[i], p); - } - if(p->mcache == nil) { - if(old==0 && i==0) - p->mcache = g->m->mcache; // bootstrap - else - p->mcache = runtime·allocmcache(); - } - } - - // redistribute runnable G's evenly - // collect all runnable goroutines in global queue preserving FIFO order - // FIFO order is required to ensure fairness even during frequent GCs - // see http://golang.org/issue/7126 - empty = false; - while(!empty) { - empty = true; - for(i = 0; i < old; i++) { - p = runtime·allp[i]; - if(p->runqhead == p->runqtail) - continue; - empty = false; - // pop from tail of local queue - p->runqtail--; - gp = p->runq[p->runqtail%nelem(p->runq)]; - // push onto head of global queue - gp->schedlink = runtime·sched.runqhead; - runtime·sched.runqhead = gp; - if(runtime·sched.runqtail == nil) - runtime·sched.runqtail = gp; - runtime·sched.runqsize++; - } - } - // fill local queues with at most nelem(p->runq)/2 goroutines - // start at 1 because current M already executes some G and will acquire allp[0] below, - // so if we have a spare G we want to put it into allp[1]. - for(i = 1; i < new * nelem(p->runq)/2 && runtime·sched.runqsize > 0; i++) { - gp = runtime·sched.runqhead; - runtime·sched.runqhead = gp->schedlink; - if(runtime·sched.runqhead == nil) - runtime·sched.runqtail = nil; - runtime·sched.runqsize--; - runqput(runtime·allp[i%new], gp); - } - - // free unused P's - for(i = new; i < old; i++) { - p = runtime·allp[i]; - runtime·freemcache(p->mcache); - p->mcache = nil; - gfpurge(p); - p->status = Pdead; - // can't free P itself because it can be referenced by an M in syscall - } - - if(g->m->p) - g->m->p->m = nil; - g->m->p = nil; - g->m->mcache = nil; - p = runtime·allp[0]; - p->m = nil; - p->status = Pidle; - acquirep(p); - for(i = new-1; i > 0; i--) { - p = runtime·allp[i]; - p->status = Pidle; - pidleput(p); - } - runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); -} - -// Associate p and the current m. -static void -acquirep(P *p) -{ - if(g->m->p || g->m->mcache) - runtime·throw("acquirep: already in go"); - if(p->m || p->status != Pidle) { - runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->m ? p->m->id : 0, p->status); - runtime·throw("acquirep: invalid p state"); - } - g->m->mcache = p->mcache; - g->m->p = p; - p->m = g->m; - p->status = Prunning; -} - -// Disassociate p and the current m. -static P* -releasep(void) -{ - P *p; - - if(g->m->p == nil || g->m->mcache == nil) - runtime·throw("releasep: invalid arg"); - p = g->m->p; - if(p->m != g->m || p->mcache != g->m->mcache || p->status != Prunning) { - runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->mcache=%p p->status=%d\n", - g->m, g->m->p, p->m, g->m->mcache, p->mcache, p->status); - runtime·throw("releasep: invalid p state"); - } - g->m->p = nil; - g->m->mcache = nil; - p->m = nil; - p->status = Pidle; - return p; -} - -static void -incidlelocked(int32 v) -{ - runtime·lock(&runtime·sched.lock); - runtime·sched.nmidlelocked += v; - if(v > 0) - checkdead(); - runtime·unlock(&runtime·sched.lock); -} - -// Check for deadlock situation. -// The check is based on number of running M's, if 0 -> deadlock. -static void -checkdead(void) -{ - G *gp; - P *p; - M *mp; - int32 run, grunning, s; - uintptr i; - - // -1 for sysmon - run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.nmidlelocked - 1; - if(run > 0) - return; - // If we are dying because of a signal caught on an already idle thread, - // freezetheworld will cause all running threads to block. - // And runtime will essentially enter into deadlock state, - // except that there is a thread that will call runtime·exit soon. - if(runtime·panicking > 0) - return; - if(run < 0) { - runtime·printf("runtime: checkdead: nmidle=%d nmidlelocked=%d mcount=%d\n", - runtime·sched.nmidle, runtime·sched.nmidlelocked, runtime·sched.mcount); - runtime·throw("checkdead: inconsistent counts"); - } - grunning = 0; - runtime·lock(&runtime·allglock); - for(i = 0; i < runtime·allglen; i++) { - gp = runtime·allg[i]; - if(gp->issystem) - continue; - s = runtime·readgstatus(gp); - switch(s&~Gscan) { - case Gwaiting: - grunning++; - break; - case Grunnable: - case Grunning: - case Gsyscall: - runtime·unlock(&runtime·allglock); - runtime·printf("runtime: checkdead: find g %D in status %d\n", gp->goid, s); - runtime·throw("checkdead: runnable g"); - break; - } - } - runtime·unlock(&runtime·allglock); - if(grunning == 0) // possible if main goroutine calls runtime·Goexit() - runtime·throw("no goroutines (main called runtime.Goexit) - deadlock!"); - - // Maybe jump time forward for playground. - if((gp = runtime·timejump()) != nil) { - runtime·casgstatus(gp, Gwaiting, Grunnable); - globrunqput(gp); - p = pidleget(); - if(p == nil) - runtime·throw("checkdead: no p for timer"); - mp = mget(); - if(mp == nil) - newm(nil, p); - else { - mp->nextp = p; - runtime·notewakeup(&mp->park); - } - return; - } - - g->m->throwing = -1; // do not dump full stacks - runtime·throw("all goroutines are asleep - deadlock!"); -} - -static void -sysmon(void) -{ - uint32 idle, delay, nscavenge; - int64 now, unixnow, lastpoll, lasttrace, lastgc; - int64 forcegcperiod, scavengelimit, lastscavenge, maxsleep; - G *gp; - - // If we go two minutes without a garbage collection, force one to run. - forcegcperiod = 2*60*1e9; - // If a heap span goes unused for 5 minutes after a garbage collection, - // we hand it back to the operating system. - scavengelimit = 5*60*1e9; - if(runtime·debug.scavenge > 0) { - // Scavenge-a-lot for testing. - forcegcperiod = 10*1e6; - scavengelimit = 20*1e6; - } - lastscavenge = runtime·nanotime(); - nscavenge = 0; - // Make wake-up period small enough for the sampling to be correct. - maxsleep = forcegcperiod/2; - if(scavengelimit < forcegcperiod) - maxsleep = scavengelimit/2; - - lasttrace = 0; - idle = 0; // how many cycles in succession we had not wokeup somebody - delay = 0; - for(;;) { - if(idle == 0) // start with 20us sleep... - delay = 20; - else if(idle > 50) // start doubling the sleep after 1ms... - delay *= 2; - if(delay > 10*1000) // up to 10ms - delay = 10*1000; - runtime·usleep(delay); - if(runtime·debug.schedtrace <= 0 && - (runtime·sched.gcwaiting || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs)) { // TODO: fast atomic - runtime·lock(&runtime·sched.lock); - if(runtime·atomicload(&runtime·sched.gcwaiting) || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) { - runtime·atomicstore(&runtime·sched.sysmonwait, 1); - runtime·unlock(&runtime·sched.lock); - runtime·notetsleep(&runtime·sched.sysmonnote, maxsleep); - runtime·lock(&runtime·sched.lock); - runtime·atomicstore(&runtime·sched.sysmonwait, 0); - runtime·noteclear(&runtime·sched.sysmonnote); - idle = 0; - delay = 20; - } - runtime·unlock(&runtime·sched.lock); - } - // poll network if not polled for more than 10ms - lastpoll = runtime·atomicload64(&runtime·sched.lastpoll); - now = runtime·nanotime(); - unixnow = runtime·unixnanotime(); - if(lastpoll != 0 && lastpoll + 10*1000*1000 < now) { - runtime·cas64(&runtime·sched.lastpoll, lastpoll, now); - gp = runtime·netpoll(false); // non-blocking - if(gp) { - // Need to decrement number of idle locked M's - // (pretending that one more is running) before injectglist. - // Otherwise it can lead to the following situation: - // injectglist grabs all P's but before it starts M's to run the P's, - // another M returns from syscall, finishes running its G, - // observes that there is no work to do and no other running M's - // and reports deadlock. - incidlelocked(-1); - injectglist(gp); - incidlelocked(1); - } - } - // retake P's blocked in syscalls - // and preempt long running G's - if(retake(now)) - idle = 0; - else - idle++; - - // check if we need to force a GC - lastgc = runtime·atomicload64(&mstats.last_gc); - if(lastgc != 0 && unixnow - lastgc > forcegcperiod && runtime·atomicload(&runtime·forcegc.idle)) { - runtime·lock(&runtime·forcegc.lock); - runtime·forcegc.idle = 0; - runtime·forcegc.g->schedlink = nil; - injectglist(runtime·forcegc.g); - runtime·unlock(&runtime·forcegc.lock); - } - - // scavenge heap once in a while - if(lastscavenge + scavengelimit/2 < now) { - runtime·MHeap_Scavenge(nscavenge, now, scavengelimit); - lastscavenge = now; - nscavenge++; - } - - if(runtime·debug.schedtrace > 0 && lasttrace + runtime·debug.schedtrace*1000000ll <= now) { - lasttrace = now; - runtime·schedtrace(runtime·debug.scheddetail); - } - } -} - -typedef struct Pdesc Pdesc; -struct Pdesc -{ - uint32 schedtick; - int64 schedwhen; - uint32 syscalltick; - int64 syscallwhen; -}; -#pragma dataflag NOPTR -static Pdesc pdesc[MaxGomaxprocs]; - -static uint32 -retake(int64 now) -{ - uint32 i, s, n; - int64 t; - P *p; - Pdesc *pd; - - n = 0; - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - if(p==nil) - continue; - pd = &pdesc[i]; - s = p->status; - if(s == Psyscall) { - // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). - t = p->syscalltick; - if(pd->syscalltick != t) { - pd->syscalltick = t; - pd->syscallwhen = now; - continue; - } - // On the one hand we don't want to retake Ps if there is no other work to do, - // but on the other hand we want to retake them eventually - // because they can prevent the sysmon thread from deep sleep. - if(p->runqhead == p->runqtail && - runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) > 0 && - pd->syscallwhen + 10*1000*1000 > now) - continue; - // Need to decrement number of idle locked M's - // (pretending that one more is running) before the CAS. - // Otherwise the M from which we retake can exit the syscall, - // increment nmidle and report deadlock. - incidlelocked(-1); - if(runtime·cas(&p->status, s, Pidle)) { - n++; - handoffp(p); - } - incidlelocked(1); - } else if(s == Prunning) { - // Preempt G if it's running for more than 10ms. - t = p->schedtick; - if(pd->schedtick != t) { - pd->schedtick = t; - pd->schedwhen = now; - continue; - } - if(pd->schedwhen + 10*1000*1000 > now) - continue; - preemptone(p); - } - } - return n; -} - -// Tell all goroutines that they have been preempted and they should stop. -// This function is purely best-effort. It can fail to inform a goroutine if a -// processor just started running it. -// No locks need to be held. -// Returns true if preemption request was issued to at least one goroutine. -static bool -preemptall(void) -{ - P *p; - int32 i; - bool res; - - res = false; - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - if(p == nil || p->status != Prunning) - continue; - res |= preemptone(p); - } - return res; -} - -// Tell the goroutine running on processor P to stop. -// This function is purely best-effort. It can incorrectly fail to inform the -// goroutine. It can send inform the wrong goroutine. Even if it informs the -// correct goroutine, that goroutine might ignore the request if it is -// simultaneously executing runtime·newstack. -// No lock needs to be held. -// Returns true if preemption request was issued. -// The actual preemption will happen at some point in the future -// and will be indicated by the gp->status no longer being -// Grunning -static bool -preemptone(P *p) -{ - M *mp; - G *gp; - - mp = p->m; - if(mp == nil || mp == g->m) - return false; - gp = mp->curg; - if(gp == nil || gp == mp->g0) - return false; - gp->preempt = true; - // Every call in a go routine checks for stack overflow by - // comparing the current stack pointer to gp->stackguard0. - // Setting gp->stackguard0 to StackPreempt folds - // preemption into the normal stack overflow check. - gp->stackguard0 = StackPreempt; - return true; -} - -void -runtime·schedtrace(bool detailed) -{ - static int64 starttime; - int64 now; - int64 id1, id2, id3; - int32 i, t, h; - uintptr gi; - int8 *fmt; - M *mp, *lockedm; - G *gp, *lockedg; - P *p; - - now = runtime·nanotime(); - if(starttime == 0) - starttime = now; - - runtime·lock(&runtime·sched.lock); - runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d spinningthreads=%d idlethreads=%d runqueue=%d", - (now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidle, runtime·sched.mcount, - runtime·sched.nmspinning, runtime·sched.nmidle, runtime·sched.runqsize); - if(detailed) { - runtime·printf(" gcwaiting=%d nmidlelocked=%d stopwait=%d sysmonwait=%d\n", - runtime·sched.gcwaiting, runtime·sched.nmidlelocked, - runtime·sched.stopwait, runtime·sched.sysmonwait); - } - // We must be careful while reading data from P's, M's and G's. - // Even if we hold schedlock, most data can be changed concurrently. - // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. - for(i = 0; i < runtime·gomaxprocs; i++) { - p = runtime·allp[i]; - if(p == nil) - continue; - mp = p->m; - h = runtime·atomicload(&p->runqhead); - t = runtime·atomicload(&p->runqtail); - if(detailed) - runtime·printf(" P%d: status=%d schedtick=%d syscalltick=%d m=%d runqsize=%d gfreecnt=%d\n", - i, p->status, p->schedtick, p->syscalltick, mp ? mp->id : -1, t-h, p->gfreecnt); - else { - // In non-detailed mode format lengths of per-P run queues as: - // [len1 len2 len3 len4] - fmt = " %d"; - if(runtime·gomaxprocs == 1) - fmt = " [%d]\n"; - else if(i == 0) - fmt = " [%d"; - else if(i == runtime·gomaxprocs-1) - fmt = " %d]\n"; - runtime·printf(fmt, t-h); - } - } - if(!detailed) { - runtime·unlock(&runtime·sched.lock); - return; - } - for(mp = runtime·allm; mp; mp = mp->alllink) { - p = mp->p; - gp = mp->curg; - lockedg = mp->lockedg; - id1 = -1; - if(p) - id1 = p->id; - id2 = -1; - if(gp) - id2 = gp->goid; - id3 = -1; - if(lockedg) - id3 = lockedg->goid; - runtime·printf(" M%d: p=%D curg=%D mallocing=%d throwing=%d gcing=%d" - " locks=%d dying=%d helpgc=%d spinning=%d blocked=%d lockedg=%D\n", - mp->id, id1, id2, - mp->mallocing, mp->throwing, mp->gcing, mp->locks, mp->dying, mp->helpgc, - mp->spinning, g->m->blocked, id3); - } - runtime·lock(&runtime·allglock); - for(gi = 0; gi < runtime·allglen; gi++) { - gp = runtime·allg[gi]; - mp = gp->m; - lockedm = gp->lockedm; - runtime·printf(" G%D: status=%d(%S) m=%d lockedm=%d\n", - gp->goid, runtime·readgstatus(gp), gp->waitreason, mp ? mp->id : -1, - lockedm ? lockedm->id : -1); - } - runtime·unlock(&runtime·allglock); - runtime·unlock(&runtime·sched.lock); -} - -// Put mp on midle list. -// Sched must be locked. -static void -mput(M *mp) -{ - mp->schedlink = runtime·sched.midle; - runtime·sched.midle = mp; - runtime·sched.nmidle++; - checkdead(); -} - -// Try to get an m from midle list. -// Sched must be locked. -static M* -mget(void) -{ - M *mp; - - if((mp = runtime·sched.midle) != nil){ - runtime·sched.midle = mp->schedlink; - runtime·sched.nmidle--; - } - return mp; -} - -// Put gp on the global runnable queue. -// Sched must be locked. -static void -globrunqput(G *gp) -{ - gp->schedlink = nil; - if(runtime·sched.runqtail) - runtime·sched.runqtail->schedlink = gp; - else - runtime·sched.runqhead = gp; - runtime·sched.runqtail = gp; - runtime·sched.runqsize++; -} - -// Put a batch of runnable goroutines on the global runnable queue. -// Sched must be locked. -static void -globrunqputbatch(G *ghead, G *gtail, int32 n) -{ - gtail->schedlink = nil; - if(runtime·sched.runqtail) - runtime·sched.runqtail->schedlink = ghead; - else - runtime·sched.runqhead = ghead; - runtime·sched.runqtail = gtail; - runtime·sched.runqsize += n; -} - -// Try get a batch of G's from the global runnable queue. -// Sched must be locked. -static G* -globrunqget(P *p, int32 max) -{ - G *gp, *gp1; - int32 n; - - if(runtime·sched.runqsize == 0) - return nil; - n = runtime·sched.runqsize/runtime·gomaxprocs+1; - if(n > runtime·sched.runqsize) - n = runtime·sched.runqsize; - if(max > 0 && n > max) - n = max; - if(n > nelem(p->runq)/2) - n = nelem(p->runq)/2; - runtime·sched.runqsize -= n; - if(runtime·sched.runqsize == 0) - runtime·sched.runqtail = nil; - gp = runtime·sched.runqhead; - runtime·sched.runqhead = gp->schedlink; - n--; - while(n--) { - gp1 = runtime·sched.runqhead; - runtime·sched.runqhead = gp1->schedlink; - runqput(p, gp1); - } - return gp; -} - -// Put p to on pidle list. -// Sched must be locked. -static void -pidleput(P *p) -{ - p->link = runtime·sched.pidle; - runtime·sched.pidle = p; - runtime·xadd(&runtime·sched.npidle, 1); // TODO: fast atomic -} - -// Try get a p from pidle list. -// Sched must be locked. -static P* -pidleget(void) -{ - P *p; - - p = runtime·sched.pidle; - if(p) { - runtime·sched.pidle = p->link; - runtime·xadd(&runtime·sched.npidle, -1); // TODO: fast atomic - } - return p; -} - -// Try to put g on local runnable queue. -// If it's full, put onto global queue. -// Executed only by the owner P. -static void -runqput(P *p, G *gp) -{ - uint32 h, t; - -retry: - h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with consumers - t = p->runqtail; - if(t - h < nelem(p->runq)) { - p->runq[t%nelem(p->runq)] = gp; - runtime·atomicstore(&p->runqtail, t+1); // store-release, makes the item available for consumption - return; - } - if(runqputslow(p, gp, h, t)) - return; - // the queue is not full, now the put above must suceed - goto retry; -} - -// Put g and a batch of work from local runnable queue on global queue. -// Executed only by the owner P. -static bool -runqputslow(P *p, G *gp, uint32 h, uint32 t) -{ - G *batch[nelem(p->runq)/2+1]; - uint32 n, i; - - // First, grab a batch from local queue. - n = t-h; - n = n/2; - if(n != nelem(p->runq)/2) - runtime·throw("runqputslow: queue is not full"); - for(i=0; i<n; i++) - batch[i] = p->runq[(h+i)%nelem(p->runq)]; - if(!runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits consume - return false; - batch[n] = gp; - // Link the goroutines. - for(i=0; i<n; i++) - batch[i]->schedlink = batch[i+1]; - // Now put the batch on global queue. - runtime·lock(&runtime·sched.lock); - globrunqputbatch(batch[0], batch[n], n+1); - runtime·unlock(&runtime·sched.lock); - return true; -} - -// Get g from local runnable queue. -// Executed only by the owner P. -static G* -runqget(P *p) -{ - G *gp; - uint32 t, h; - - for(;;) { - h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with other consumers - t = p->runqtail; - if(t == h) - return nil; - gp = p->runq[h%nelem(p->runq)]; - if(runtime·cas(&p->runqhead, h, h+1)) // cas-release, commits consume - return gp; - } -} - -// Grabs a batch of goroutines from local runnable queue. -// batch array must be of size nelem(p->runq)/2. Returns number of grabbed goroutines. -// Can be executed by any P. -static uint32 -runqgrab(P *p, G **batch) -{ - uint32 t, h, n, i; - - for(;;) { - h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with other consumers - t = runtime·atomicload(&p->runqtail); // load-acquire, synchronize with the producer - n = t-h; - n = n - n/2; - if(n == 0) - break; - if(n > nelem(p->runq)/2) // read inconsistent h and t - continue; - for(i=0; i<n; i++) - batch[i] = p->runq[(h+i)%nelem(p->runq)]; - if(runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits consume - break; - } - return n; -} - -// Steal half of elements from local runnable queue of p2 -// and put onto local runnable queue of p. -// Returns one of the stolen elements (or nil if failed). -static G* -runqsteal(P *p, P *p2) -{ - G *gp; - G *batch[nelem(p->runq)/2]; - uint32 t, h, n, i; - - n = runqgrab(p2, batch); - if(n == 0) - return nil; - n--; - gp = batch[n]; - if(n == 0) - return gp; - h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with consumers - t = p->runqtail; - if(t - h + n >= nelem(p->runq)) - runtime·throw("runqsteal: runq overflow"); - for(i=0; i<n; i++, t++) - p->runq[t%nelem(p->runq)] = batch[i]; - runtime·atomicstore(&p->runqtail, t); // store-release, makes the item available for consumption - return gp; -} - -void -runtime·testSchedLocalQueue(void) -{ - P *p; - G *gs; - int32 i, j; - - p = (P*)runtime·mallocgc(sizeof(*p), nil, FlagNoScan); - gs = (G*)runtime·mallocgc(nelem(p->runq)*sizeof(*gs), nil, FlagNoScan); - - for(i = 0; i < nelem(p->runq); i++) { - if(runqget(p) != nil) - runtime·throw("runq is not empty initially"); - for(j = 0; j < i; j++) - runqput(p, &gs[i]); - for(j = 0; j < i; j++) { - if(runqget(p) != &gs[i]) { - runtime·printf("bad element at iter %d/%d\n", i, j); - runtime·throw("bad element"); - } - } - if(runqget(p) != nil) - runtime·throw("runq is not empty afterwards"); - } -} - -void -runtime·testSchedLocalQueueSteal(void) -{ - P *p1, *p2; - G *gs, *gp; - int32 i, j, s; - - p1 = (P*)runtime·mallocgc(sizeof(*p1), nil, FlagNoScan); - p2 = (P*)runtime·mallocgc(sizeof(*p2), nil, FlagNoScan); - gs = (G*)runtime·mallocgc(nelem(p1->runq)*sizeof(*gs), nil, FlagNoScan); - - for(i = 0; i < nelem(p1->runq); i++) { - for(j = 0; j < i; j++) { - gs[j].sig = 0; - runqput(p1, &gs[j]); - } - gp = runqsteal(p2, p1); - s = 0; - if(gp) { - s++; - gp->sig++; - } - while(gp = runqget(p2)) { - s++; - gp->sig++; - } - while(gp = runqget(p1)) - gp->sig++; - for(j = 0; j < i; j++) { - if(gs[j].sig != 1) { - runtime·printf("bad element %d(%d) at iter %d\n", j, gs[j].sig, i); - runtime·throw("bad element"); - } - } - if(s != i/2 && s != i/2+1) { - runtime·printf("bad steal %d, want %d or %d, iter %d\n", - s, i/2, i/2+1, i); - runtime·throw("bad steal"); - } - } -} - -void -runtime·setmaxthreads_m(void) -{ - int32 in; - int32 out; - - in = g->m->scalararg[0]; - - runtime·lock(&runtime·sched.lock); - out = runtime·sched.maxmcount; - runtime·sched.maxmcount = in; - checkmcount(); - runtime·unlock(&runtime·sched.lock); - - g->m->scalararg[0] = out; -} - -static int8 experiment[] = GOEXPERIMENT; // defined in zaexperiment.h - -static bool -haveexperiment(int8 *name) -{ - int32 i, j; - - for(i=0; i<sizeof(experiment); i++) { - if((i == 0 || experiment[i-1] == ',') && experiment[i] == name[0]) { - for(j=0; name[j]; j++) - if(experiment[i+j] != name[j]) - goto nomatch; - if(experiment[i+j] != '\0' && experiment[i+j] != ',') - goto nomatch; - return 1; - } - nomatch:; - } - return 0; -} - -#pragma textflag NOSPLIT -void -sync·runtime_procPin(intptr p) -{ - M *mp; - - mp = g->m; - // Disable preemption. - mp->locks++; - p = mp->p->id; - FLUSH(&p); -} - -#pragma textflag NOSPLIT -void -sync·runtime_procUnpin() -{ - g->m->locks--; -} diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 517ca03df..50920afe8 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -6,8 +6,6 @@ package runtime import "unsafe" -func newsysmon() - func runtime_init() func main_init() func main_main() @@ -29,7 +27,7 @@ func main() { maxstacksize = 250000000 } - onM(newsysmon) + systemstack(newsysmon) // Lock the main goroutine onto this, the main OS thread, // during initialization. Most programs won't care, but a few @@ -55,6 +53,24 @@ func main() { memstats.enablegc = true // now that runtime is initialized, GC is okay + if iscgo { + if _cgo_thread_start == nil { + gothrow("_cgo_thread_start missing") + } + if _cgo_malloc == nil { + gothrow("_cgo_malloc missing") + } + if _cgo_free == nil { + gothrow("_cgo_free missing") + } + if _cgo_setenv == nil { + gothrow("_cgo_setenv missing") + } + if _cgo_unsetenv == nil { + gothrow("_cgo_unsetenv missing") + } + } + main_init() needUnlock = false @@ -80,8 +96,6 @@ func main() { } } -var parkunlock_c byte - // start forcegc helper goroutine func init() { go forcegchelper() @@ -115,7 +129,7 @@ func Gosched() { // Puts the current goroutine into a waiting state and calls unlockf. // If unlockf returns false, the goroutine is resumed. -func gopark(unlockf unsafe.Pointer, lock unsafe.Pointer, reason string) { +func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string) { mp := acquirem() gp := mp.curg status := readgstatus(gp) @@ -123,7 +137,7 @@ func gopark(unlockf unsafe.Pointer, lock unsafe.Pointer, reason string) { gothrow("gopark: bad g status") } mp.waitlock = lock - mp.waitunlockf = unlockf + mp.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf)) gp.waitreason = reason releasem(mp) // can't do anything that might move the G between Ms here. @@ -133,14 +147,13 @@ func gopark(unlockf unsafe.Pointer, lock unsafe.Pointer, reason string) { // Puts the current goroutine into a waiting state and unlocks the lock. // The goroutine can be made runnable again by calling goready(gp). func goparkunlock(lock *mutex, reason string) { - gopark(unsafe.Pointer(&parkunlock_c), unsafe.Pointer(lock), reason) + gopark(parkunlock_c, unsafe.Pointer(lock), reason) } func goready(gp *g) { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(gp) - onM(ready_m) - releasem(mp) + systemstack(func() { + ready(gp) + }) } //go:nosplit @@ -233,6 +246,11 @@ func newG() *g { return new(g) } +var ( + allgs []*g + allglock mutex +) + func allgadd(gp *g) { if readgstatus(gp) == _Gidle { gothrow("allgadd: bad status Gidle") diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go new file mode 100644 index 000000000..81b211d0d --- /dev/null +++ b/src/runtime/proc1.go @@ -0,0 +1,3170 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +var ( + m0 m + g0 g +) + +// Goroutine scheduler +// The scheduler's job is to distribute ready-to-run goroutines over worker threads. +// +// The main concepts are: +// G - goroutine. +// M - worker thread, or machine. +// P - processor, a resource that is required to execute Go code. +// M must have an associated P to execute Go code, however it can be +// blocked or in a syscall w/o an associated P. +// +// Design doc at http://golang.org/s/go11sched. + +const ( + // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once. + // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. + _GoidCacheBatch = 16 +) + +/* +SchedT sched; +int32 gomaxprocs; +uint32 needextram; +bool iscgo; +M m0; +G g0; // idle goroutine for m0 +G* lastg; +M* allm; +M* extram; +P* allp[MaxGomaxprocs+1]; +int8* goos; +int32 ncpu; +int32 newprocs; + +Mutex allglock; // the following vars are protected by this lock or by stoptheworld +G** allg; +Slice allgs; +uintptr allglen; +ForceGCState forcegc; + +void mstart(void); +static void runqput(P*, G*); +static G* runqget(P*); +static bool runqputslow(P*, G*, uint32, uint32); +static G* runqsteal(P*, P*); +static void mput(M*); +static M* mget(void); +static void mcommoninit(M*); +static void schedule(void); +static void procresize(int32); +static void acquirep(P*); +static P* releasep(void); +static void newm(void(*)(void), P*); +static void stopm(void); +static void startm(P*, bool); +static void handoffp(P*); +static void wakep(void); +static void stoplockedm(void); +static void startlockedm(G*); +static void sysmon(void); +static uint32 retake(int64); +static void incidlelocked(int32); +static void checkdead(void); +static void exitsyscall0(G*); +void park_m(G*); +static void goexit0(G*); +static void gfput(P*, G*); +static G* gfget(P*); +static void gfpurge(P*); +static void globrunqput(G*); +static void globrunqputbatch(G*, G*, int32); +static G* globrunqget(P*, int32); +static P* pidleget(void); +static void pidleput(P*); +static void injectglist(G*); +static bool preemptall(void); +static bool preemptone(P*); +static bool exitsyscallfast(void); +static bool haveexperiment(int8*); +void allgadd(G*); +static void dropg(void); + +extern String buildVersion; +*/ + +// The bootstrap sequence is: +// +// call osinit +// call schedinit +// make & queue new G +// call runtime·mstart +// +// The new G calls runtime·main. +func schedinit() { + // raceinit must be the first call to race detector. + // In particular, it must be done before mallocinit below calls racemapshadow. + _g_ := getg() + if raceenabled { + _g_.racectx = raceinit() + } + + sched.maxmcount = 10000 + + tracebackinit() + symtabinit() + stackinit() + mallocinit() + mcommoninit(_g_.m) + + goargs() + goenvs() + parsedebugvars() + gcinit() + + sched.lastpoll = uint64(nanotime()) + procs := 1 + if n := goatoi(gogetenv("GOMAXPROCS")); n > 0 { + if n > _MaxGomaxprocs { + n = _MaxGomaxprocs + } + procs = n + } + procresize(int32(procs)) + + if buildVersion == "" { + // Condition should never trigger. This code just serves + // to ensure runtime·buildVersion is kept in the resulting binary. + buildVersion = "unknown" + } +} + +func newsysmon() { + _newm(sysmon, nil) +} + +func dumpgstatus(gp *g) { + _g_ := getg() + print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") + print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n") +} + +func checkmcount() { + // sched lock is held + if sched.mcount > sched.maxmcount { + print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n") + gothrow("thread exhaustion") + } +} + +func mcommoninit(mp *m) { + _g_ := getg() + + // g0 stack won't make sense for user (and is not necessary unwindable). + if _g_ != _g_.m.g0 { + callers(1, &mp.createstack[0], len(mp.createstack)) + } + + mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks()) + if mp.fastrand == 0 { + mp.fastrand = 0x49f6428a + } + + lock(&sched.lock) + mp.id = sched.mcount + sched.mcount++ + checkmcount() + mpreinit(mp) + if mp.gsignal != nil { + mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard + } + + // Add to allm so garbage collector doesn't free g->m + // when it is just in a register or thread-local storage. + mp.alllink = allm + + // NumCgoCall() iterates over allm w/o schedlock, + // so we need to publish it safely. + atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp)) + unlock(&sched.lock) +} + +// Mark gp ready to run. +func ready(gp *g) { + status := readgstatus(gp) + + // Mark runnable. + _g_ := getg() + _g_.m.locks++ // disable preemption because it can be holding p in a local var + if status&^_Gscan != _Gwaiting { + dumpgstatus(gp) + gothrow("bad g->status in ready") + } + + // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq + casgstatus(gp, _Gwaiting, _Grunnable) + runqput(_g_.m.p, gp) + if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic + wakep() + } + _g_.m.locks-- + if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } +} + +func gcprocs() int32 { + // Figure out how many CPUs to use during GC. + // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. + lock(&sched.lock) + n := gomaxprocs + if n > ncpu { + n = ncpu + } + if n > _MaxGcproc { + n = _MaxGcproc + } + if n > sched.nmidle+1 { // one M is currently running + n = sched.nmidle + 1 + } + unlock(&sched.lock) + return n +} + +func needaddgcproc() bool { + lock(&sched.lock) + n := gomaxprocs + if n > ncpu { + n = ncpu + } + if n > _MaxGcproc { + n = _MaxGcproc + } + n -= sched.nmidle + 1 // one M is currently running + unlock(&sched.lock) + return n > 0 +} + +func helpgc(nproc int32) { + _g_ := getg() + lock(&sched.lock) + pos := 0 + for n := int32(1); n < nproc; n++ { // one M is currently running + if allp[pos].mcache == _g_.m.mcache { + pos++ + } + mp := mget() + if mp == nil { + gothrow("gcprocs inconsistency") + } + mp.helpgc = n + mp.mcache = allp[pos].mcache + pos++ + notewakeup(&mp.park) + } + unlock(&sched.lock) +} + +// Similar to stoptheworld but best-effort and can be called several times. +// There is no reverse operation, used during crashing. +// This function must not lock any mutexes. +func freezetheworld() { + if gomaxprocs == 1 { + return + } + // stopwait and preemption requests can be lost + // due to races with concurrently executing threads, + // so try several times + for i := 0; i < 5; i++ { + // this should tell the scheduler to not start any new goroutines + sched.stopwait = 0x7fffffff + atomicstore(&sched.gcwaiting, 1) + // this should stop running goroutines + if !preemptall() { + break // no running goroutines + } + usleep(1000) + } + // to be sure + usleep(1000) + preemptall() + usleep(1000) +} + +func isscanstatus(status uint32) bool { + if status == _Gscan { + gothrow("isscanstatus: Bad status Gscan") + } + return status&_Gscan == _Gscan +} + +// All reads and writes of g's status go through readgstatus, casgstatus +// castogscanstatus, casfrom_Gscanstatus. +//go:nosplit +func readgstatus(gp *g) uint32 { + return atomicload(&gp.atomicstatus) +} + +// The Gscanstatuses are acting like locks and this releases them. +// If it proves to be a performance hit we should be able to make these +// simple atomic stores but for now we are going to throw if +// we see an inconsistent state. +func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { + success := false + + // Check that transition is valid. + switch oldval { + case _Gscanrunnable, + _Gscanwaiting, + _Gscanrunning, + _Gscansyscall: + if newval == oldval&^_Gscan { + success = cas(&gp.atomicstatus, oldval, newval) + } + case _Gscanenqueue: + if newval == _Gwaiting { + success = cas(&gp.atomicstatus, oldval, newval) + } + } + if !success { + print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") + dumpgstatus(gp) + gothrow("casfrom_Gscanstatus: gp->status is not in scan state") + } +} + +// This will return false if the gp is not in the expected status and the cas fails. +// This acts like a lock acquire while the casfromgstatus acts like a lock release. +func castogscanstatus(gp *g, oldval, newval uint32) bool { + switch oldval { + case _Grunnable, + _Gwaiting, + _Gsyscall: + if newval == oldval|_Gscan { + return cas(&gp.atomicstatus, oldval, newval) + } + case _Grunning: + if newval == _Gscanrunning || newval == _Gscanenqueue { + return cas(&gp.atomicstatus, oldval, newval) + } + } + print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n") + gothrow("castogscanstatus") + panic("not reached") +} + +// If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus +// and casfrom_Gscanstatus instead. +// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that +// put it in the Gscan state is finished. +//go:nosplit +func casgstatus(gp *g, oldval, newval uint32) { + if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { + systemstack(func() { + print("casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") + gothrow("casgstatus: bad incoming values") + }) + } + + // loop if gp->atomicstatus is in a scan state giving + // GC time to finish and change the state to oldval. + for !cas(&gp.atomicstatus, oldval, newval) { + // Help GC if needed. + if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) { + gp.preemptscan = false + systemstack(func() { + gcphasework(gp) + }) + } + } +} + +// stopg ensures that gp is stopped at a GC safe point where its stack can be scanned +// or in the context of a moving collector the pointers can be flipped from pointing +// to old object to pointing to new objects. +// If stopg returns true, the caller knows gp is at a GC safe point and will remain there until +// the caller calls restartg. +// If stopg returns false, the caller is not responsible for calling restartg. This can happen +// if another thread, either the gp itself or another GC thread is taking the responsibility +// to do the GC work related to this thread. +func stopg(gp *g) bool { + for { + if gp.gcworkdone { + return false + } + + switch s := readgstatus(gp); s { + default: + dumpgstatus(gp) + gothrow("stopg: gp->atomicstatus is not valid") + + case _Gdead: + return false + + case _Gcopystack: + // Loop until a new stack is in place. + + case _Grunnable, + _Gsyscall, + _Gwaiting: + // Claim goroutine by setting scan bit. + if !castogscanstatus(gp, s, s|_Gscan) { + break + } + // In scan state, do work. + gcphasework(gp) + return true + + case _Gscanrunnable, + _Gscanwaiting, + _Gscansyscall: + // Goroutine already claimed by another GC helper. + return false + + case _Grunning: + // Claim goroutine, so we aren't racing with a status + // transition away from Grunning. + if !castogscanstatus(gp, _Grunning, _Gscanrunning) { + break + } + + // Mark gp for preemption. + if !gp.gcworkdone { + gp.preemptscan = true + gp.preempt = true + gp.stackguard0 = stackPreempt + } + + // Unclaim. + casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning) + return false + } + } +} + +// The GC requests that this routine be moved from a scanmumble state to a mumble state. +func restartg(gp *g) { + s := readgstatus(gp) + switch s { + default: + dumpgstatus(gp) + gothrow("restartg: unexpected status") + + case _Gdead: + // ok + + case _Gscanrunnable, + _Gscanwaiting, + _Gscansyscall: + casfrom_Gscanstatus(gp, s, s&^_Gscan) + + // Scan is now completed. + // Goroutine now needs to be made runnable. + // We put it on the global run queue; ready blocks on the global scheduler lock. + case _Gscanenqueue: + casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting) + if gp != getg().m.curg { + gothrow("processing Gscanenqueue on wrong m") + } + dropg() + ready(gp) + } +} + +func stopscanstart(gp *g) { + _g_ := getg() + if _g_ == gp { + gothrow("GC not moved to G0") + } + if stopg(gp) { + if !isscanstatus(readgstatus(gp)) { + dumpgstatus(gp) + gothrow("GC not in scan state") + } + restartg(gp) + } +} + +// Runs on g0 and does the actual work after putting the g back on the run queue. +func mquiesce(gpmaster *g) { + activeglen := len(allgs) + // enqueue the calling goroutine. + restartg(gpmaster) + for i := 0; i < activeglen; i++ { + gp := allgs[i] + if readgstatus(gp) == _Gdead { + gp.gcworkdone = true // noop scan. + } else { + gp.gcworkdone = false + } + stopscanstart(gp) + } + + // Check that the G's gcwork (such as scanning) has been done. If not do it now. + // You can end up doing work here if the page trap on a Grunning Goroutine has + // not been sprung or in some race situations. For example a runnable goes dead + // and is started up again with a gp->gcworkdone set to false. + for i := 0; i < activeglen; i++ { + gp := allgs[i] + for !gp.gcworkdone { + status := readgstatus(gp) + if status == _Gdead { + //do nothing, scan not needed. + gp.gcworkdone = true // scan is a noop + break + } + if status == _Grunning && gp.stackguard0 == uintptr(stackPreempt) && notetsleep(&sched.stopnote, 100*1000) { // nanosecond arg + noteclear(&sched.stopnote) + } else { + stopscanstart(gp) + } + } + } + + for i := 0; i < activeglen; i++ { + gp := allgs[i] + status := readgstatus(gp) + if isscanstatus(status) { + print("mstopandscang:bottom: post scan bad status gp=", gp, " has status ", hex(status), "\n") + dumpgstatus(gp) + } + if !gp.gcworkdone && status != _Gdead { + print("mstopandscang:bottom: post scan gp=", gp, "->gcworkdone still false\n") + dumpgstatus(gp) + } + } + + schedule() // Never returns. +} + +// quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point. +// If the global gcphase is GCmark quiesce will ensure that all of the goroutine's stacks +// have been scanned before it returns. +func quiesce(mastergp *g) { + castogscanstatus(mastergp, _Grunning, _Gscanenqueue) + // Now move this to the g0 (aka m) stack. + // g0 will potentially scan this thread and put mastergp on the runqueue + mcall(mquiesce) +} + +// This is used by the GC as well as the routines that do stack dumps. In the case +// of GC all the routines can be reliably stopped. This is not always the case +// when the system is in panic or being exited. +func stoptheworld() { + _g_ := getg() + + // If we hold a lock, then we won't be able to stop another M + // that is blocked trying to acquire the lock. + if _g_.m.locks > 0 { + gothrow("stoptheworld: holding locks") + } + + lock(&sched.lock) + sched.stopwait = gomaxprocs + atomicstore(&sched.gcwaiting, 1) + preemptall() + // stop current P + _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic. + sched.stopwait-- + // try to retake all P's in Psyscall status + for i := 0; i < int(gomaxprocs); i++ { + p := allp[i] + s := p.status + if s == _Psyscall && cas(&p.status, s, _Pgcstop) { + sched.stopwait-- + } + } + // stop idle P's + for { + p := pidleget() + if p == nil { + break + } + p.status = _Pgcstop + sched.stopwait-- + } + wait := sched.stopwait > 0 + unlock(&sched.lock) + + // wait for remaining P's to stop voluntarily + if wait { + for { + // wait for 100us, then try to re-preempt in case of any races + if notetsleep(&sched.stopnote, 100*1000) { + noteclear(&sched.stopnote) + break + } + preemptall() + } + } + if sched.stopwait != 0 { + gothrow("stoptheworld: not stopped") + } + for i := 0; i < int(gomaxprocs); i++ { + p := allp[i] + if p.status != _Pgcstop { + gothrow("stoptheworld: not stopped") + } + } +} + +func mhelpgc() { + _g_ := getg() + _g_.m.helpgc = -1 +} + +func starttheworld() { + _g_ := getg() + + _g_.m.locks++ // disable preemption because it can be holding p in a local var + gp := netpoll(false) // non-blocking + injectglist(gp) + add := needaddgcproc() + lock(&sched.lock) + if newprocs != 0 { + procresize(newprocs) + newprocs = 0 + } else { + procresize(gomaxprocs) + } + sched.gcwaiting = 0 + + var p1 *p + for { + p := pidleget() + if p == nil { + break + } + // procresize() puts p's with work at the beginning of the list. + // Once we reach a p without a run queue, the rest don't have one either. + if p.runqhead == p.runqtail { + pidleput(p) + break + } + p.m = mget() + p.link = p1 + p1 = p + } + if sched.sysmonwait != 0 { + sched.sysmonwait = 0 + notewakeup(&sched.sysmonnote) + } + unlock(&sched.lock) + + for p1 != nil { + p := p1 + p1 = p1.link + if p.m != nil { + mp := p.m + p.m = nil + if mp.nextp != nil { + gothrow("starttheworld: inconsistent mp->nextp") + } + mp.nextp = p + notewakeup(&mp.park) + } else { + // Start M to run P. Do not start another M below. + _newm(nil, p) + add = false + } + } + + if add { + // If GC could have used another helper proc, start one now, + // in the hope that it will be available next time. + // It would have been even better to start it before the collection, + // but doing so requires allocating memory, so it's tricky to + // coordinate. This lazy approach works out in practice: + // we don't mind if the first couple gc rounds don't have quite + // the maximum number of procs. + _newm(mhelpgc, nil) + } + _g_.m.locks-- + if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } +} + +// Called to start an M. +//go:nosplit +func mstart() { + _g_ := getg() + + if _g_.stack.lo == 0 { + // Initialize stack bounds from system stack. + // Cgo may have left stack size in stack.hi. + size := _g_.stack.hi + if size == 0 { + size = 8192 + } + _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size))) + _g_.stack.lo = _g_.stack.hi - size + 1024 + } + // Initialize stack guards so that we can start calling + // both Go and C functions with stack growth prologues. + _g_.stackguard0 = _g_.stack.lo + _StackGuard + _g_.stackguard1 = _g_.stackguard0 + mstart1() +} + +func mstart1() { + _g_ := getg() + + if _g_ != _g_.m.g0 { + gothrow("bad runtime·mstart") + } + + // Record top of stack for use by mcall. + // Once we call schedule we're never coming back, + // so other calls can reuse this stack space. + gosave(&_g_.m.g0.sched) + _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used + asminit() + minit() + + // Install signal handlers; after minit so that minit can + // prepare the thread to be able to handle the signals. + if _g_.m == &m0 { + initsig() + } + + if _g_.m.mstartfn != nil { + fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn)) + fn() + } + + if _g_.m.helpgc != 0 { + _g_.m.helpgc = 0 + stopm() + } else if _g_.m != &m0 { + acquirep(_g_.m.nextp) + _g_.m.nextp = nil + } + schedule() + + // TODO(brainman): This point is never reached, because scheduler + // does not release os threads at the moment. But once this path + // is enabled, we must remove our seh here. +} + +// When running with cgo, we call _cgo_thread_start +// to start threads for us so that we can play nicely with +// foreign code. +var cgoThreadStart unsafe.Pointer + +type cgothreadstart struct { + g *g + tls *uint64 + fn unsafe.Pointer +} + +// Allocate a new m unassociated with any thread. +// Can use p for allocation context if needed. +func allocm(_p_ *p) *m { + _g_ := getg() + _g_.m.locks++ // disable GC because it can be called from sysmon + if _g_.m.p == nil { + acquirep(_p_) // temporarily borrow p for mallocs in this function + } + mp := newM() + mcommoninit(mp) + + // In case of cgo or Solaris, pthread_create will make us a stack. + // Windows and Plan 9 will layout sched stack on OS stack. + if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" { + mp.g0 = malg(-1) + } else { + mp.g0 = malg(8192) + } + mp.g0.m = mp + + if _p_ == _g_.m.p { + releasep() + } + _g_.m.locks-- + if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } + + return mp +} + +func allocg() *g { + return newG() +} + +// needm is called when a cgo callback happens on a +// thread without an m (a thread not created by Go). +// In this case, needm is expected to find an m to use +// and return with m, g initialized correctly. +// Since m and g are not set now (likely nil, but see below) +// needm is limited in what routines it can call. In particular +// it can only call nosplit functions (textflag 7) and cannot +// do any scheduling that requires an m. +// +// In order to avoid needing heavy lifting here, we adopt +// the following strategy: there is a stack of available m's +// that can be stolen. Using compare-and-swap +// to pop from the stack has ABA races, so we simulate +// a lock by doing an exchange (via casp) to steal the stack +// head and replace the top pointer with MLOCKED (1). +// This serves as a simple spin lock that we can use even +// without an m. The thread that locks the stack in this way +// unlocks the stack by storing a valid stack head pointer. +// +// In order to make sure that there is always an m structure +// available to be stolen, we maintain the invariant that there +// is always one more than needed. At the beginning of the +// program (if cgo is in use) the list is seeded with a single m. +// If needm finds that it has taken the last m off the list, its job +// is - once it has installed its own m so that it can do things like +// allocate memory - to create a spare m and put it on the list. +// +// Each of these extra m's also has a g0 and a curg that are +// pressed into service as the scheduling stack and current +// goroutine for the duration of the cgo callback. +// +// When the callback is done with the m, it calls dropm to +// put the m back on the list. +//go:nosplit +func needm(x byte) { + if needextram != 0 { + // Can happen if C/C++ code calls Go from a global ctor. + // Can not throw, because scheduler is not initialized yet. + // XXX + // write(2, unsafe.Pointer("fatal error: cgo callback before cgo call\n"), sizeof("fatal error: cgo callback before cgo call\n") - 1) + exit(1) + } + + // Lock extra list, take head, unlock popped list. + // nilokay=false is safe here because of the invariant above, + // that the extra list always contains or will soon contain + // at least one m. + mp := lockextra(false) + + // Set needextram when we've just emptied the list, + // so that the eventual call into cgocallbackg will + // allocate a new m for the extra list. We delay the + // allocation until then so that it can be done + // after exitsyscall makes sure it is okay to be + // running at all (that is, there's no garbage collection + // running right now). + mp.needextram = mp.schedlink == nil + unlockextra(mp.schedlink) + + // Install g (= m->g0) and set the stack bounds + // to match the current stack. We don't actually know + // how big the stack is, like we don't know how big any + // scheduling stack is, but we assume there's at least 32 kB, + // which is more than enough for us. + setg(mp.g0) + _g_ := getg() + _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024 + _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024 + _g_.stackguard0 = _g_.stack.lo + _StackGuard + + // Initialize this thread to use the m. + asminit() + minit() +} + +// newextram allocates an m and puts it on the extra list. +// It is called with a working local m, so that it can do things +// like call schedlock and allocate. +func newextram() { + // Create extra goroutine locked to extra m. + // The goroutine is the context in which the cgo callback will run. + // The sched.pc will never be returned to, but setting it to + // goexit makes clear to the traceback routines where + // the goroutine stack ends. + mp := allocm(nil) + gp := malg(4096) + gp.sched.pc = funcPC(goexit) + _PCQuantum + gp.sched.sp = gp.stack.hi + gp.sched.sp -= 4 * regSize // extra space in case of reads slightly beyond frame + gp.sched.lr = 0 + gp.sched.g = gp + gp.syscallpc = gp.sched.pc + gp.syscallsp = gp.sched.sp + // malg returns status as Gidle, change to Gsyscall before adding to allg + // where GC will see it. + casgstatus(gp, _Gidle, _Gsyscall) + gp.m = mp + mp.curg = gp + mp.locked = _LockInternal + mp.lockedg = gp + gp.lockedm = mp + gp.goid = int64(xadd64(&sched.goidgen, 1)) + if raceenabled { + gp.racectx = racegostart(funcPC(newextram)) + } + // put on allg for garbage collector + allgadd(gp) + + // Add m to the extra list. + mnext := lockextra(true) + mp.schedlink = mnext + unlockextra(mp) +} + +// dropm is called when a cgo callback has called needm but is now +// done with the callback and returning back into the non-Go thread. +// It puts the current m back onto the extra list. +// +// The main expense here is the call to signalstack to release the +// m's signal stack, and then the call to needm on the next callback +// from this thread. It is tempting to try to save the m for next time, +// which would eliminate both these costs, but there might not be +// a next time: the current thread (which Go does not control) might exit. +// If we saved the m for that thread, there would be an m leak each time +// such a thread exited. Instead, we acquire and release an m on each +// call. These should typically not be scheduling operations, just a few +// atomics, so the cost should be small. +// +// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread +// variable using pthread_key_create. Unlike the pthread keys we already use +// on OS X, this dummy key would never be read by Go code. It would exist +// only so that we could register at thread-exit-time destructor. +// That destructor would put the m back onto the extra list. +// This is purely a performance optimization. The current version, +// in which dropm happens on each cgo call, is still correct too. +// We may have to keep the current version on systems with cgo +// but without pthreads, like Windows. +func dropm() { + // Undo whatever initialization minit did during needm. + unminit() + + // Clear m and g, and return m to the extra list. + // After the call to setmg we can only call nosplit functions. + mp := getg().m + setg(nil) + + mnext := lockextra(true) + mp.schedlink = mnext + unlockextra(mp) +} + +var extram uintptr + +// lockextra locks the extra list and returns the list head. +// The caller must unlock the list by storing a new list head +// to extram. If nilokay is true, then lockextra will +// return a nil list head if that's what it finds. If nilokay is false, +// lockextra will keep waiting until the list head is no longer nil. +//go:nosplit +func lockextra(nilokay bool) *m { + const locked = 1 + + for { + old := atomicloaduintptr(&extram) + if old == locked { + yield := osyield + yield() + continue + } + if old == 0 && !nilokay { + usleep(1) + continue + } + if casuintptr(&extram, old, locked) { + return (*m)(unsafe.Pointer(old)) + } + yield := osyield + yield() + continue + } +} + +//go:nosplit +func unlockextra(mp *m) { + atomicstoreuintptr(&extram, uintptr(unsafe.Pointer(mp))) +} + +// Create a new m. It will start off with a call to fn, or else the scheduler. +func _newm(fn func(), _p_ *p) { + mp := allocm(_p_) + mp.nextp = _p_ + mp.mstartfn = *(*unsafe.Pointer)(unsafe.Pointer(&fn)) + + if iscgo { + var ts cgothreadstart + if _cgo_thread_start == nil { + gothrow("_cgo_thread_start missing") + } + ts.g = mp.g0 + ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) + ts.fn = unsafe.Pointer(funcPC(mstart)) + asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) + return + } + newosproc(mp, unsafe.Pointer(mp.g0.stack.hi)) +} + +// Stops execution of the current m until new work is available. +// Returns with acquired P. +func stopm() { + _g_ := getg() + + if _g_.m.locks != 0 { + gothrow("stopm holding locks") + } + if _g_.m.p != nil { + gothrow("stopm holding p") + } + if _g_.m.spinning { + _g_.m.spinning = false + xadd(&sched.nmspinning, -1) + } + +retry: + lock(&sched.lock) + mput(_g_.m) + unlock(&sched.lock) + notesleep(&_g_.m.park) + noteclear(&_g_.m.park) + if _g_.m.helpgc != 0 { + gchelper() + _g_.m.helpgc = 0 + _g_.m.mcache = nil + goto retry + } + acquirep(_g_.m.nextp) + _g_.m.nextp = nil +} + +func mspinning() { + getg().m.spinning = true +} + +// Schedules some M to run the p (creates an M if necessary). +// If p==nil, tries to get an idle P, if no idle P's does nothing. +func startm(_p_ *p, spinning bool) { + lock(&sched.lock) + if _p_ == nil { + _p_ = pidleget() + if _p_ == nil { + unlock(&sched.lock) + if spinning { + xadd(&sched.nmspinning, -1) + } + return + } + } + mp := mget() + unlock(&sched.lock) + if mp == nil { + var fn func() + if spinning { + fn = mspinning + } + _newm(fn, _p_) + return + } + if mp.spinning { + gothrow("startm: m is spinning") + } + if mp.nextp != nil { + gothrow("startm: m has p") + } + mp.spinning = spinning + mp.nextp = _p_ + notewakeup(&mp.park) +} + +// Hands off P from syscall or locked M. +func handoffp(_p_ *p) { + // if it has local work, start it straight away + if _p_.runqhead != _p_.runqtail || sched.runqsize != 0 { + startm(_p_, false) + return + } + // no local work, check that there are no spinning/idle M's, + // otherwise our help is not required + if atomicload(&sched.nmspinning)+atomicload(&sched.npidle) == 0 && cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic + startm(_p_, true) + return + } + lock(&sched.lock) + if sched.gcwaiting != 0 { + _p_.status = _Pgcstop + sched.stopwait-- + if sched.stopwait == 0 { + notewakeup(&sched.stopnote) + } + unlock(&sched.lock) + return + } + if sched.runqsize != 0 { + unlock(&sched.lock) + startm(_p_, false) + return + } + // If this is the last running P and nobody is polling network, + // need to wakeup another M to poll network. + if sched.npidle == uint32(gomaxprocs-1) && atomicload64(&sched.lastpoll) != 0 { + unlock(&sched.lock) + startm(_p_, false) + return + } + pidleput(_p_) + unlock(&sched.lock) +} + +// Tries to add one more P to execute G's. +// Called when a G is made runnable (newproc, ready). +func wakep() { + // be conservative about spinning threads + if !cas(&sched.nmspinning, 0, 1) { + return + } + startm(nil, true) +} + +// Stops execution of the current m that is locked to a g until the g is runnable again. +// Returns with acquired P. +func stoplockedm() { + _g_ := getg() + + if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m { + gothrow("stoplockedm: inconsistent locking") + } + if _g_.m.p != nil { + // Schedule another M to run this p. + _p_ := releasep() + handoffp(_p_) + } + incidlelocked(1) + // Wait until another thread schedules lockedg again. + notesleep(&_g_.m.park) + noteclear(&_g_.m.park) + status := readgstatus(_g_.m.lockedg) + if status&^_Gscan != _Grunnable { + print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n") + dumpgstatus(_g_) + gothrow("stoplockedm: not runnable") + } + acquirep(_g_.m.nextp) + _g_.m.nextp = nil +} + +// Schedules the locked m to run the locked gp. +func startlockedm(gp *g) { + _g_ := getg() + + mp := gp.lockedm + if mp == _g_.m { + gothrow("startlockedm: locked to me") + } + if mp.nextp != nil { + gothrow("startlockedm: m has p") + } + // directly handoff current P to the locked m + incidlelocked(-1) + _p_ := releasep() + mp.nextp = _p_ + notewakeup(&mp.park) + stopm() +} + +// Stops the current m for stoptheworld. +// Returns when the world is restarted. +func gcstopm() { + _g_ := getg() + + if sched.gcwaiting == 0 { + gothrow("gcstopm: not waiting for gc") + } + if _g_.m.spinning { + _g_.m.spinning = false + xadd(&sched.nmspinning, -1) + } + _p_ := releasep() + lock(&sched.lock) + _p_.status = _Pgcstop + sched.stopwait-- + if sched.stopwait == 0 { + notewakeup(&sched.stopnote) + } + unlock(&sched.lock) + stopm() +} + +// Schedules gp to run on the current M. +// Never returns. +func execute(gp *g) { + _g_ := getg() + + casgstatus(gp, _Grunnable, _Grunning) + gp.waitsince = 0 + gp.preempt = false + gp.stackguard0 = gp.stack.lo + _StackGuard + _g_.m.p.schedtick++ + _g_.m.curg = gp + gp.m = _g_.m + + // Check whether the profiler needs to be turned on or off. + hz := sched.profilehz + if _g_.m.profilehz != hz { + resetcpuprofiler(hz) + } + + gogo(&gp.sched) +} + +// Finds a runnable goroutine to execute. +// Tries to steal from other P's, get g from global queue, poll network. +func findrunnable() *g { + _g_ := getg() + +top: + if sched.gcwaiting != 0 { + gcstopm() + goto top + } + if fingwait && fingwake { + if gp := wakefing(); gp != nil { + ready(gp) + } + } + + // local runq + if gp := runqget(_g_.m.p); gp != nil { + return gp + } + + // global runq + if sched.runqsize != 0 { + lock(&sched.lock) + gp := globrunqget(_g_.m.p, 0) + unlock(&sched.lock) + if gp != nil { + return gp + } + } + + // poll network - returns list of goroutines + if gp := netpoll(false); gp != nil { // non-blocking + injectglist(gp.schedlink) + casgstatus(gp, _Gwaiting, _Grunnable) + return gp + } + + // If number of spinning M's >= number of busy P's, block. + // This is necessary to prevent excessive CPU consumption + // when GOMAXPROCS>>1 but the program parallelism is low. + if !_g_.m.spinning && 2*atomicload(&sched.nmspinning) >= uint32(gomaxprocs)-atomicload(&sched.npidle) { // TODO: fast atomic + goto stop + } + if !_g_.m.spinning { + _g_.m.spinning = true + xadd(&sched.nmspinning, 1) + } + // random steal from other P's + for i := 0; i < int(2*gomaxprocs); i++ { + if sched.gcwaiting != 0 { + goto top + } + _p_ := allp[fastrand1()%uint32(gomaxprocs)] + var gp *g + if _p_ == _g_.m.p { + gp = runqget(_p_) + } else { + gp = runqsteal(_g_.m.p, _p_) + } + if gp != nil { + return gp + } + } +stop: + + // return P and block + lock(&sched.lock) + if sched.gcwaiting != 0 { + unlock(&sched.lock) + goto top + } + if sched.runqsize != 0 { + gp := globrunqget(_g_.m.p, 0) + unlock(&sched.lock) + return gp + } + _p_ := releasep() + pidleput(_p_) + unlock(&sched.lock) + if _g_.m.spinning { + _g_.m.spinning = false + xadd(&sched.nmspinning, -1) + } + + // check all runqueues once again + for i := 0; i < int(gomaxprocs); i++ { + _p_ := allp[i] + if _p_ != nil && _p_.runqhead != _p_.runqtail { + lock(&sched.lock) + _p_ = pidleget() + unlock(&sched.lock) + if _p_ != nil { + acquirep(_p_) + goto top + } + break + } + } + + // poll network + if xchg64(&sched.lastpoll, 0) != 0 { + if _g_.m.p != nil { + gothrow("findrunnable: netpoll with p") + } + if _g_.m.spinning { + gothrow("findrunnable: netpoll with spinning") + } + gp := netpoll(true) // block until new work is available + atomicstore64(&sched.lastpoll, uint64(nanotime())) + if gp != nil { + lock(&sched.lock) + _p_ = pidleget() + unlock(&sched.lock) + if _p_ != nil { + acquirep(_p_) + injectglist(gp.schedlink) + casgstatus(gp, _Gwaiting, _Grunnable) + return gp + } + injectglist(gp) + } + } + stopm() + goto top +} + +func resetspinning() { + _g_ := getg() + + var nmspinning uint32 + if _g_.m.spinning { + _g_.m.spinning = false + nmspinning = xadd(&sched.nmspinning, -1) + if nmspinning < 0 { + gothrow("findrunnable: negative nmspinning") + } + } else { + nmspinning = atomicload(&sched.nmspinning) + } + + // M wakeup policy is deliberately somewhat conservative (see nmspinning handling), + // so see if we need to wakeup another P here. + if nmspinning == 0 && atomicload(&sched.npidle) > 0 { + wakep() + } +} + +// Injects the list of runnable G's into the scheduler. +// Can run concurrently with GC. +func injectglist(glist *g) { + if glist == nil { + return + } + lock(&sched.lock) + var n int + for n = 0; glist != nil; n++ { + gp := glist + glist = gp.schedlink + casgstatus(gp, _Gwaiting, _Grunnable) + globrunqput(gp) + } + unlock(&sched.lock) + for ; n != 0 && sched.npidle != 0; n-- { + startm(nil, false) + } +} + +// One round of scheduler: find a runnable goroutine and execute it. +// Never returns. +func schedule() { + _g_ := getg() + + if _g_.m.locks != 0 { + gothrow("schedule: holding locks") + } + + if _g_.m.lockedg != nil { + stoplockedm() + execute(_g_.m.lockedg) // Never returns. + } + +top: + if sched.gcwaiting != 0 { + gcstopm() + goto top + } + + var gp *g + // Check the global runnable queue once in a while to ensure fairness. + // Otherwise two goroutines can completely occupy the local runqueue + // by constantly respawning each other. + tick := _g_.m.p.schedtick + // This is a fancy way to say tick%61==0, + // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors. + if uint64(tick)-((uint64(tick)*0x4325c53f)>>36)*61 == 0 && sched.runqsize > 0 { + lock(&sched.lock) + gp = globrunqget(_g_.m.p, 1) + unlock(&sched.lock) + if gp != nil { + resetspinning() + } + } + if gp == nil { + gp = runqget(_g_.m.p) + if gp != nil && _g_.m.spinning { + gothrow("schedule: spinning with local work") + } + } + if gp == nil { + gp = findrunnable() // blocks until work is available + resetspinning() + } + + if gp.lockedm != nil { + // Hands off own p to the locked m, + // then blocks waiting for a new p. + startlockedm(gp) + goto top + } + + execute(gp) +} + +// dropg removes the association between m and the current goroutine m->curg (gp for short). +// Typically a caller sets gp's status away from Grunning and then +// immediately calls dropg to finish the job. The caller is also responsible +// for arranging that gp will be restarted using ready at an +// appropriate time. After calling dropg and arranging for gp to be +// readied later, the caller can do other work but eventually should +// call schedule to restart the scheduling of goroutines on this m. +func dropg() { + _g_ := getg() + + if _g_.m.lockedg == nil { + _g_.m.curg.m = nil + _g_.m.curg = nil + } +} + +// Puts the current goroutine into a waiting state and calls unlockf. +// If unlockf returns false, the goroutine is resumed. +func park(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string) { + _g_ := getg() + + _g_.m.waitlock = lock + _g_.m.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf)) + _g_.waitreason = reason + mcall(park_m) +} + +func parkunlock_c(gp *g, lock unsafe.Pointer) bool { + unlock((*mutex)(lock)) + return true +} + +// Puts the current goroutine into a waiting state and unlocks the lock. +// The goroutine can be made runnable again by calling ready(gp). +func parkunlock(lock *mutex, reason string) { + park(parkunlock_c, unsafe.Pointer(lock), reason) +} + +// park continuation on g0. +func park_m(gp *g) { + _g_ := getg() + + casgstatus(gp, _Grunning, _Gwaiting) + dropg() + + if _g_.m.waitunlockf != nil { + fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf)) + ok := fn(gp, _g_.m.waitlock) + _g_.m.waitunlockf = nil + _g_.m.waitlock = nil + if !ok { + casgstatus(gp, _Gwaiting, _Grunnable) + execute(gp) // Schedule it back, never returns. + } + } + schedule() +} + +// Gosched continuation on g0. +func gosched_m(gp *g) { + status := readgstatus(gp) + if status&^_Gscan != _Grunning { + dumpgstatus(gp) + gothrow("bad g status") + } + casgstatus(gp, _Grunning, _Grunnable) + dropg() + lock(&sched.lock) + globrunqput(gp) + unlock(&sched.lock) + + schedule() +} + +// Finishes execution of the current goroutine. +// Must be NOSPLIT because it is called from Go. (TODO - probably not anymore) +//go:nosplit +func goexit1() { + if raceenabled { + racegoend() + } + mcall(goexit0) +} + +// goexit continuation on g0. +func goexit0(gp *g) { + _g_ := getg() + + casgstatus(gp, _Grunning, _Gdead) + gp.m = nil + gp.lockedm = nil + _g_.m.lockedg = nil + gp.paniconfault = false + gp._defer = nil // should be true already but just in case. + gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data. + gp.writebuf = nil + gp.waitreason = "" + gp.param = nil + + dropg() + + if _g_.m.locked&^_LockExternal != 0 { + print("invalid m->locked = ", _g_.m.locked, "\n") + gothrow("internal lockOSThread error") + } + _g_.m.locked = 0 + gfput(_g_.m.p, gp) + schedule() +} + +//go:nosplit +func save(pc, sp uintptr) { + _g_ := getg() + + _g_.sched.pc = pc + _g_.sched.sp = sp + _g_.sched.lr = 0 + _g_.sched.ret = 0 + _g_.sched.ctxt = nil + _g_.sched.g = _g_ +} + +// The goroutine g is about to enter a system call. +// Record that it's not using the cpu anymore. +// This is called only from the go syscall library and cgocall, +// not from the low-level system calls used by the +// +// Entersyscall cannot split the stack: the gosave must +// make g->sched refer to the caller's stack segment, because +// entersyscall is going to return immediately after. +// +// Nothing entersyscall calls can split the stack either. +// We cannot safely move the stack during an active call to syscall, +// because we do not know which of the uintptr arguments are +// really pointers (back into the stack). +// In practice, this means that we make the fast path run through +// entersyscall doing no-split things, and the slow path has to use systemstack +// to run bigger things on the system stack. +// +// reentersyscall is the entry point used by cgo callbacks, where explicitly +// saved SP and PC are restored. This is needed when exitsyscall will be called +// from a function further up in the call stack than the parent, as g->syscallsp +// must always point to a valid stack frame. entersyscall below is the normal +// entry point for syscalls, which obtains the SP and PC from the caller. +//go:nosplit +func reentersyscall(pc, sp uintptr) { + _g_ := getg() + + // Disable preemption because during this function g is in Gsyscall status, + // but can have inconsistent g->sched, do not let GC observe it. + _g_.m.locks++ + + // Entersyscall must not call any function that might split/grow the stack. + // (See details in comment above.) + // Catch calls that might, by replacing the stack guard with something that + // will trip any stack check and leaving a flag to tell newstack to die. + _g_.stackguard0 = stackPreempt + _g_.throwsplit = true + + // Leave SP around for GC and traceback. + save(pc, sp) + _g_.syscallsp = sp + _g_.syscallpc = pc + casgstatus(_g_, _Grunning, _Gsyscall) + if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { + systemstack(entersyscall_bad) + } + + if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic + systemstack(entersyscall_sysmon) + save(pc, sp) + } + + _g_.m.mcache = nil + _g_.m.p.m = nil + atomicstore(&_g_.m.p.status, _Psyscall) + if sched.gcwaiting != 0 { + systemstack(entersyscall_gcwait) + save(pc, sp) + } + + // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). + // We set _StackGuard to StackPreempt so that first split stack check calls morestack. + // Morestack detects this case and throws. + _g_.stackguard0 = stackPreempt + _g_.m.locks-- +} + +// Standard syscall entry used by the go syscall library and normal cgo calls. +//go:nosplit +func entersyscall(dummy int32) { + reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) +} + +func entersyscall_bad() { + var gp *g + gp = getg().m.curg + print("entersyscall inconsistent ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + gothrow("entersyscall") +} + +func entersyscall_sysmon() { + lock(&sched.lock) + if atomicload(&sched.sysmonwait) != 0 { + atomicstore(&sched.sysmonwait, 0) + notewakeup(&sched.sysmonnote) + } + unlock(&sched.lock) +} + +func entersyscall_gcwait() { + _g_ := getg() + + lock(&sched.lock) + if sched.stopwait > 0 && cas(&_g_.m.p.status, _Psyscall, _Pgcstop) { + if sched.stopwait--; sched.stopwait == 0 { + notewakeup(&sched.stopnote) + } + } + unlock(&sched.lock) +} + +// The same as entersyscall(), but with a hint that the syscall is blocking. +//go:nosplit +func entersyscallblock(dummy int32) { + _g_ := getg() + + _g_.m.locks++ // see comment in entersyscall + _g_.throwsplit = true + _g_.stackguard0 = stackPreempt // see comment in entersyscall + + // Leave SP around for GC and traceback. + save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) + _g_.syscallsp = _g_.sched.sp + _g_.syscallpc = _g_.sched.pc + casgstatus(_g_, _Grunning, _Gsyscall) + if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { + systemstack(entersyscall_bad) + } + + systemstack(entersyscallblock_handoff) + + // Resave for traceback during blocked call. + save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) + + _g_.m.locks-- +} + +func entersyscallblock_handoff() { + handoffp(releasep()) +} + +// The goroutine g exited its system call. +// Arrange for it to run on a cpu again. +// This is called only from the go syscall library, not +// from the low-level system calls used by the +//go:nosplit +func exitsyscall(dummy int32) { + _g_ := getg() + + _g_.m.locks++ // see comment in entersyscall + if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp { + gothrow("exitsyscall: syscall frame is no longer valid") + } + + _g_.waitsince = 0 + if exitsyscallfast() { + if _g_.m.mcache == nil { + gothrow("lost mcache") + } + // There's a cpu for us, so we can run. + _g_.m.p.syscalltick++ + // We need to cas the status and scan before resuming... + casgstatus(_g_, _Gsyscall, _Grunning) + + // Garbage collector isn't running (since we are), + // so okay to clear syscallsp. + _g_.syscallsp = 0 + _g_.m.locks-- + if _g_.preempt { + // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } else { + // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock + _g_.stackguard0 = _g_.stack.lo + _StackGuard + } + _g_.throwsplit = false + return + } + + _g_.m.locks-- + + // Call the scheduler. + mcall(exitsyscall0) + + if _g_.m.mcache == nil { + gothrow("lost mcache") + } + + // Scheduler returned, so we're allowed to run now. + // Delete the syscallsp information that we left for + // the garbage collector during the system call. + // Must wait until now because until gosched returns + // we don't know for sure that the garbage collector + // is not running. + _g_.syscallsp = 0 + _g_.m.p.syscalltick++ + _g_.throwsplit = false +} + +//go:nosplit +func exitsyscallfast() bool { + _g_ := getg() + + // Freezetheworld sets stopwait but does not retake P's. + if sched.stopwait != 0 { + _g_.m.p = nil + return false + } + + // Try to re-acquire the last P. + if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) { + // There's a cpu for us, so we can run. + _g_.m.mcache = _g_.m.p.mcache + _g_.m.p.m = _g_.m + return true + } + + // Try to get any other idle P. + _g_.m.p = nil + if sched.pidle != nil { + var ok bool + systemstack(func() { + ok = exitsyscallfast_pidle() + }) + if ok { + return true + } + } + return false +} + +func exitsyscallfast_pidle() bool { + lock(&sched.lock) + _p_ := pidleget() + if _p_ != nil && atomicload(&sched.sysmonwait) != 0 { + atomicstore(&sched.sysmonwait, 0) + notewakeup(&sched.sysmonnote) + } + unlock(&sched.lock) + if _p_ != nil { + acquirep(_p_) + return true + } + return false +} + +// exitsyscall slow path on g0. +// Failed to acquire P, enqueue gp as runnable. +func exitsyscall0(gp *g) { + _g_ := getg() + + casgstatus(gp, _Gsyscall, _Grunnable) + dropg() + lock(&sched.lock) + _p_ := pidleget() + if _p_ == nil { + globrunqput(gp) + } else if atomicload(&sched.sysmonwait) != 0 { + atomicstore(&sched.sysmonwait, 0) + notewakeup(&sched.sysmonnote) + } + unlock(&sched.lock) + if _p_ != nil { + acquirep(_p_) + execute(gp) // Never returns. + } + if _g_.m.lockedg != nil { + // Wait until another thread schedules gp and so m again. + stoplockedm() + execute(gp) // Never returns. + } + stopm() + schedule() // Never returns. +} + +func beforefork() { + gp := getg().m.curg + + // Fork can hang if preempted with signals frequently enough (see issue 5517). + // Ensure that we stay on the same M where we disable profiling. + gp.m.locks++ + if gp.m.profilehz != 0 { + resetcpuprofiler(0) + } + + // This function is called before fork in syscall package. + // Code between fork and exec must not allocate memory nor even try to grow stack. + // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack. + // runtime_AfterFork will undo this in parent process, but not in child. + gp.stackguard0 = stackFork +} + +// Called from syscall package before fork. +//go:nosplit +func syscall_BeforeFork() { + systemstack(beforefork) +} + +func afterfork() { + gp := getg().m.curg + + // See the comment in beforefork. + gp.stackguard0 = gp.stack.lo + _StackGuard + + hz := sched.profilehz + if hz != 0 { + resetcpuprofiler(hz) + } + gp.m.locks-- +} + +// Called from syscall package after fork in parent. +//go:nosplit +func syscall_AfterFork() { + systemstack(afterfork) +} + +// Allocate a new g, with a stack big enough for stacksize bytes. +func malg(stacksize int32) *g { + newg := allocg() + if stacksize >= 0 { + stacksize = round2(_StackSystem + stacksize) + systemstack(func() { + newg.stack = stackalloc(uint32(stacksize)) + }) + newg.stackguard0 = newg.stack.lo + _StackGuard + newg.stackguard1 = ^uintptr(0) + } + return newg +} + +// Create a new g running fn with siz bytes of arguments. +// Put it on the queue of g's waiting to run. +// The compiler turns a go statement into a call to this. +// Cannot split the stack because it assumes that the arguments +// are available sequentially after &fn; they would not be +// copied if a stack split occurred. +//go:nosplit +func newproc(siz int32, fn *funcval) { + argp := add(unsafe.Pointer(&fn), ptrSize) + if hasLinkRegister { + argp = add(argp, ptrSize) // skip caller's saved LR + } + + pc := getcallerpc(unsafe.Pointer(&siz)) + systemstack(func() { + newproc1(fn, (*uint8)(argp), siz, 0, pc) + }) +} + +// Create a new g running fn with narg bytes of arguments starting +// at argp and returning nret bytes of results. callerpc is the +// address of the go statement that created this. The new g is put +// on the queue of g's waiting to run. +func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g { + _g_ := getg() + + if fn == nil { + _g_.m.throwing = -1 // do not dump full stacks + gothrow("go of nil func value") + } + _g_.m.locks++ // disable preemption because it can be holding p in a local var + siz := narg + nret + siz = (siz + 7) &^ 7 + + // We could allocate a larger initial stack if necessary. + // Not worth it: this is almost always an error. + // 4*sizeof(uintreg): extra space added below + // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). + if siz >= _StackMin-4*regSize-regSize { + gothrow("newproc: function arguments too large for new goroutine") + } + + _p_ := _g_.m.p + newg := gfget(_p_) + if newg == nil { + newg = malg(_StackMin) + casgstatus(newg, _Gidle, _Gdead) + allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. + } + if newg.stack.hi == 0 { + gothrow("newproc1: newg missing stack") + } + + if readgstatus(newg) != _Gdead { + gothrow("newproc1: new g is not Gdead") + } + + sp := newg.stack.hi + sp -= 4 * regSize // extra space in case of reads slightly beyond frame + sp -= uintptr(siz) + memmove(unsafe.Pointer(sp), unsafe.Pointer(argp), uintptr(narg)) + if hasLinkRegister { + // caller's LR + sp -= ptrSize + *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil + } + + memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched)) + newg.sched.sp = sp + newg.sched.pc = funcPC(goexit) + _PCQuantum // +PCQuantum so that previous instruction is in same function + newg.sched.g = newg + gostartcallfn(&newg.sched, fn) + newg.gopc = callerpc + casgstatus(newg, _Gdead, _Grunnable) + + if _p_.goidcache == _p_.goidcacheend { + // Sched.goidgen is the last allocated id, + // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. + // At startup sched.goidgen=0, so main goroutine receives goid=1. + _p_.goidcache = xadd64(&sched.goidgen, _GoidCacheBatch) + _p_.goidcache -= _GoidCacheBatch - 1 + _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch + } + newg.goid = int64(_p_.goidcache) + _p_.goidcache++ + if raceenabled { + newg.racectx = racegostart(callerpc) + } + runqput(_p_, newg) + + if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic + wakep() + } + _g_.m.locks-- + if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } + return newg +} + +// Put on gfree list. +// If local list is too long, transfer a batch to the global list. +func gfput(_p_ *p, gp *g) { + if readgstatus(gp) != _Gdead { + gothrow("gfput: bad status (not Gdead)") + } + + stksize := gp.stack.hi - gp.stack.lo + + if stksize != _FixedStack { + // non-standard stack size - free it. + stackfree(gp.stack) + gp.stack.lo = 0 + gp.stack.hi = 0 + gp.stackguard0 = 0 + } + + gp.schedlink = _p_.gfree + _p_.gfree = gp + _p_.gfreecnt++ + if _p_.gfreecnt >= 64 { + lock(&sched.gflock) + for _p_.gfreecnt >= 32 { + _p_.gfreecnt-- + gp = _p_.gfree + _p_.gfree = gp.schedlink + gp.schedlink = sched.gfree + sched.gfree = gp + sched.ngfree++ + } + unlock(&sched.gflock) + } +} + +// Get from gfree list. +// If local list is empty, grab a batch from global list. +func gfget(_p_ *p) *g { +retry: + gp := _p_.gfree + if gp == nil && sched.gfree != nil { + lock(&sched.gflock) + for _p_.gfreecnt < 32 && sched.gfree != nil { + _p_.gfreecnt++ + gp = sched.gfree + sched.gfree = gp.schedlink + sched.ngfree-- + gp.schedlink = _p_.gfree + _p_.gfree = gp + } + unlock(&sched.gflock) + goto retry + } + if gp != nil { + _p_.gfree = gp.schedlink + _p_.gfreecnt-- + if gp.stack.lo == 0 { + // Stack was deallocated in gfput. Allocate a new one. + systemstack(func() { + gp.stack = stackalloc(_FixedStack) + }) + gp.stackguard0 = gp.stack.lo + _StackGuard + } else { + if raceenabled { + racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) + } + } + } + return gp +} + +// Purge all cached G's from gfree list to the global list. +func gfpurge(_p_ *p) { + lock(&sched.gflock) + for _p_.gfreecnt != 0 { + _p_.gfreecnt-- + gp := _p_.gfree + _p_.gfree = gp.schedlink + gp.schedlink = sched.gfree + sched.gfree = gp + sched.ngfree++ + } + unlock(&sched.gflock) +} + +// Breakpoint executes a breakpoint trap. +func Breakpoint() { + breakpoint() +} + +// dolockOSThread is called by LockOSThread and lockOSThread below +// after they modify m.locked. Do not allow preemption during this call, +// or else the m might be different in this function than in the caller. +//go:nosplit +func dolockOSThread() { + _g_ := getg() + _g_.m.lockedg = _g_ + _g_.lockedm = _g_.m +} + +//go:nosplit + +// LockOSThread wires the calling goroutine to its current operating system thread. +// Until the calling goroutine exits or calls UnlockOSThread, it will always +// execute in that thread, and no other goroutine can. +func LockOSThread() { + getg().m.locked |= _LockExternal + dolockOSThread() +} + +//go:nosplit +func lockOSThread() { + getg().m.locked += _LockInternal + dolockOSThread() +} + +// dounlockOSThread is called by UnlockOSThread and unlockOSThread below +// after they update m->locked. Do not allow preemption during this call, +// or else the m might be in different in this function than in the caller. +//go:nosplit +func dounlockOSThread() { + _g_ := getg() + if _g_.m.locked != 0 { + return + } + _g_.m.lockedg = nil + _g_.lockedm = nil +} + +//go:nosplit + +// UnlockOSThread unwires the calling goroutine from its fixed operating system thread. +// If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op. +func UnlockOSThread() { + getg().m.locked &^= _LockExternal + dounlockOSThread() +} + +//go:nosplit +func unlockOSThread() { + _g_ := getg() + if _g_.m.locked < _LockInternal { + systemstack(badunlockosthread) + } + _g_.m.locked -= _LockInternal + dounlockOSThread() +} + +func badunlockosthread() { + gothrow("runtime: internal error: misuse of lockOSThread/unlockOSThread") +} + +func gcount() int32 { + n := int32(allglen) - sched.ngfree + for i := 0; ; i++ { + _p_ := allp[i] + if _p_ == nil { + break + } + n -= _p_.gfreecnt + } + + // All these variables can be changed concurrently, so the result can be inconsistent. + // But at least the current goroutine is running. + if n < 1 { + n = 1 + } + return n +} + +func mcount() int32 { + return sched.mcount +} + +var prof struct { + lock uint32 + hz int32 +} + +func _System() { _System() } +func _ExternalCode() { _ExternalCode() } +func _GC() { _GC() } + +var etext struct{} + +// Called if we receive a SIGPROF signal. +func sigprof(pc *uint8, sp *uint8, lr *uint8, gp *g, mp *m) { + var n int32 + var traceback bool + var stk [100]uintptr + + if prof.hz == 0 { + return + } + + // Profiling runs concurrently with GC, so it must not allocate. + mp.mallocing++ + + // Define that a "user g" is a user-created goroutine, and a "system g" + // is one that is m->g0 or m->gsignal. We've only made sure that we + // can unwind user g's, so exclude the system g's. + // + // It is not quite as easy as testing gp == m->curg (the current user g) + // because we might be interrupted for profiling halfway through a + // goroutine switch. The switch involves updating three (or four) values: + // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, + // because once it gets updated the new g is running. + // + // When switching from a user g to a system g, LR is not considered live, + // so the update only affects g, SP, and PC. Since PC must be last, there + // the possible partial transitions in ordinary execution are (1) g alone is updated, + // (2) both g and SP are updated, and (3) SP alone is updated. + // If g is updated, we'll see a system g and not look closer. + // If SP alone is updated, we can detect the partial transition by checking + // whether the SP is within g's stack bounds. (We could also require that SP + // be changed only after g, but the stack bounds check is needed by other + // cases, so there is no need to impose an additional requirement.) + // + // There is one exceptional transition to a system g, not in ordinary execution. + // When a signal arrives, the operating system starts the signal handler running + // with an updated PC and SP. The g is updated last, at the beginning of the + // handler. There are two reasons this is okay. First, until g is updated the + // g and SP do not match, so the stack bounds check detects the partial transition. + // Second, signal handlers currently run with signals disabled, so a profiling + // signal cannot arrive during the handler. + // + // When switching from a system g to a user g, there are three possibilities. + // + // First, it may be that the g switch has no PC update, because the SP + // either corresponds to a user g throughout (as in asmcgocall) + // or because it has been arranged to look like a user g frame + // (as in cgocallback_gofunc). In this case, since the entire + // transition is a g+SP update, a partial transition updating just one of + // those will be detected by the stack bounds check. + // + // Second, when returning from a signal handler, the PC and SP updates + // are performed by the operating system in an atomic update, so the g + // update must be done before them. The stack bounds check detects + // the partial transition here, and (again) signal handlers run with signals + // disabled, so a profiling signal cannot arrive then anyway. + // + // Third, the common case: it may be that the switch updates g, SP, and PC + // separately, as in gogo. + // + // Because gogo is the only instance, we check whether the PC lies + // within that function, and if so, not ask for a traceback. This approach + // requires knowing the size of the gogo function, which we + // record in arch_*.h and check in runtime_test.go. + // + // There is another apparently viable approach, recorded here in case + // the "PC within gogo" check turns out not to be usable. + // It would be possible to delay the update of either g or SP until immediately + // before the PC update instruction. Then, because of the stack bounds check, + // the only problematic interrupt point is just before that PC update instruction, + // and the sigprof handler can detect that instruction and simulate stepping past + // it in order to reach a consistent state. On ARM, the update of g must be made + // in two places (in R10 and also in a TLS slot), so the delayed update would + // need to be the SP update. The sigprof handler must read the instruction at + // the current PC and if it was the known instruction (for example, JMP BX or + // MOV R2, PC), use that other register in place of the PC value. + // The biggest drawback to this solution is that it requires that we can tell + // whether it's safe to read from the memory pointed at by PC. + // In a correct program, we can test PC == nil and otherwise read, + // but if a profiling signal happens at the instant that a program executes + // a bad jump (before the program manages to handle the resulting fault) + // the profiling handler could fault trying to read nonexistent memory. + // + // To recap, there are no constraints on the assembly being used for the + // transition. We simply require that g and SP match and that the PC is not + // in gogo. + traceback = true + usp := uintptr(unsafe.Pointer(sp)) + gogo := funcPC(gogo) + if gp == nil || gp != mp.curg || + usp < gp.stack.lo || gp.stack.hi < usp || + (gogo <= uintptr(unsafe.Pointer(pc)) && uintptr(unsafe.Pointer(pc)) < gogo+_RuntimeGogoBytes) { + traceback = false + } + + n = 0 + if traceback { + n = int32(gentraceback(uintptr(unsafe.Pointer(pc)), uintptr(unsafe.Pointer(sp)), uintptr(unsafe.Pointer(lr)), gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap)) + } + if !traceback || n <= 0 { + // Normal traceback is impossible or has failed. + // See if it falls into several common cases. + n = 0 + if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { + // Cgo, we can't unwind and symbolize arbitrary C code, + // so instead collect Go stack that leads to the cgo call. + // This is especially important on windows, since all syscalls are cgo calls. + n = int32(gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0)) + } + if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 { + // Libcall, i.e. runtime syscall on windows. + // Collect Go stack that leads to the call. + n = int32(gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0)) + } + if n == 0 { + // If all of the above has failed, account it against abstract "System" or "GC". + n = 2 + // "ExternalCode" is better than "etext". + if uintptr(unsafe.Pointer(pc)) > uintptr(unsafe.Pointer(&etext)) { + pc = (*uint8)(unsafe.Pointer(uintptr(funcPC(_ExternalCode) + _PCQuantum))) + } + stk[0] = uintptr(unsafe.Pointer(pc)) + if mp.gcing != 0 || mp.helpgc != 0 { + stk[1] = funcPC(_GC) + _PCQuantum + } else { + stk[1] = funcPC(_System) + _PCQuantum + } + } + } + + if prof.hz != 0 { + // Simple cas-lock to coordinate with setcpuprofilerate. + for !cas(&prof.lock, 0, 1) { + osyield() + } + if prof.hz != 0 { + cpuproftick(&stk[0], n) + } + atomicstore(&prof.lock, 0) + } + mp.mallocing-- +} + +// Arrange to call fn with a traceback hz times a second. +func setcpuprofilerate_m(hz int32) { + // Force sane arguments. + if hz < 0 { + hz = 0 + } + + // Disable preemption, otherwise we can be rescheduled to another thread + // that has profiling enabled. + _g_ := getg() + _g_.m.locks++ + + // Stop profiler on this thread so that it is safe to lock prof. + // if a profiling signal came in while we had prof locked, + // it would deadlock. + resetcpuprofiler(0) + + for !cas(&prof.lock, 0, 1) { + osyield() + } + prof.hz = hz + atomicstore(&prof.lock, 0) + + lock(&sched.lock) + sched.profilehz = hz + unlock(&sched.lock) + + if hz != 0 { + resetcpuprofiler(hz) + } + + _g_.m.locks-- +} + +// Change number of processors. The world is stopped, sched is locked. +func procresize(new int32) { + old := gomaxprocs + if old < 0 || old > _MaxGomaxprocs || new <= 0 || new > _MaxGomaxprocs { + gothrow("procresize: invalid arg") + } + + // initialize new P's + for i := int32(0); i < new; i++ { + p := allp[i] + if p == nil { + p = newP() + p.id = i + p.status = _Pgcstop + atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(p)) + } + if p.mcache == nil { + if old == 0 && i == 0 { + if getg().m.mcache == nil { + gothrow("missing mcache?") + } + p.mcache = getg().m.mcache // bootstrap + } else { + p.mcache = allocmcache() + } + } + } + + // redistribute runnable G's evenly + // collect all runnable goroutines in global queue preserving FIFO order + // FIFO order is required to ensure fairness even during frequent GCs + // see http://golang.org/issue/7126 + empty := false + for !empty { + empty = true + for i := int32(0); i < old; i++ { + p := allp[i] + if p.runqhead == p.runqtail { + continue + } + empty = false + // pop from tail of local queue + p.runqtail-- + gp := p.runq[p.runqtail%uint32(len(p.runq))] + // push onto head of global queue + gp.schedlink = sched.runqhead + sched.runqhead = gp + if sched.runqtail == nil { + sched.runqtail = gp + } + sched.runqsize++ + } + } + + // fill local queues with at most len(p.runq)/2 goroutines + // start at 1 because current M already executes some G and will acquire allp[0] below, + // so if we have a spare G we want to put it into allp[1]. + var _p_ p + for i := int32(1); i < new*int32(len(_p_.runq))/2 && sched.runqsize > 0; i++ { + gp := sched.runqhead + sched.runqhead = gp.schedlink + if sched.runqhead == nil { + sched.runqtail = nil + } + sched.runqsize-- + runqput(allp[i%new], gp) + } + + // free unused P's + for i := new; i < old; i++ { + p := allp[i] + freemcache(p.mcache) + p.mcache = nil + gfpurge(p) + p.status = _Pdead + // can't free P itself because it can be referenced by an M in syscall + } + + _g_ := getg() + if _g_.m.p != nil { + _g_.m.p.m = nil + } + _g_.m.p = nil + _g_.m.mcache = nil + p := allp[0] + p.m = nil + p.status = _Pidle + acquirep(p) + for i := new - 1; i > 0; i-- { + p := allp[i] + p.status = _Pidle + pidleput(p) + } + var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32 + atomicstore((*uint32)(unsafe.Pointer(int32p)), uint32(new)) +} + +// Associate p and the current m. +func acquirep(_p_ *p) { + _g_ := getg() + + if _g_.m.p != nil || _g_.m.mcache != nil { + gothrow("acquirep: already in go") + } + if _p_.m != nil || _p_.status != _Pidle { + id := int32(0) + if _p_.m != nil { + id = _p_.m.id + } + print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") + gothrow("acquirep: invalid p state") + } + _g_.m.mcache = _p_.mcache + _g_.m.p = _p_ + _p_.m = _g_.m + _p_.status = _Prunning +} + +// Disassociate p and the current m. +func releasep() *p { + _g_ := getg() + + if _g_.m.p == nil || _g_.m.mcache == nil { + gothrow("releasep: invalid arg") + } + _p_ := _g_.m.p + if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { + print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") + gothrow("releasep: invalid p state") + } + _g_.m.p = nil + _g_.m.mcache = nil + _p_.m = nil + _p_.status = _Pidle + return _p_ +} + +func incidlelocked(v int32) { + lock(&sched.lock) + sched.nmidlelocked += v + if v > 0 { + checkdead() + } + unlock(&sched.lock) +} + +// Check for deadlock situation. +// The check is based on number of running M's, if 0 -> deadlock. +func checkdead() { + // If we are dying because of a signal caught on an already idle thread, + // freezetheworld will cause all running threads to block. + // And runtime will essentially enter into deadlock state, + // except that there is a thread that will call exit soon. + if panicking > 0 { + return + } + + // -1 for sysmon + run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1 + if run > 0 { + return + } + if run < 0 { + print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n") + gothrow("checkdead: inconsistent counts") + } + + grunning := 0 + lock(&allglock) + for i := 0; i < len(allgs); i++ { + gp := allgs[i] + if gp.issystem { + continue + } + s := readgstatus(gp) + switch s &^ _Gscan { + case _Gwaiting: + grunning++ + case _Grunnable, + _Grunning, + _Gsyscall: + unlock(&allglock) + print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n") + gothrow("checkdead: runnable g") + } + } + unlock(&allglock) + if grunning == 0 { // possible if main goroutine calls runtime·Goexit() + gothrow("no goroutines (main called runtime.Goexit) - deadlock!") + } + + // Maybe jump time forward for playground. + gp := timejump() + if gp != nil { + casgstatus(gp, _Gwaiting, _Grunnable) + globrunqput(gp) + _p_ := pidleget() + if _p_ == nil { + gothrow("checkdead: no p for timer") + } + mp := mget() + if mp == nil { + _newm(nil, _p_) + } else { + mp.nextp = _p_ + notewakeup(&mp.park) + } + return + } + + getg().m.throwing = -1 // do not dump full stacks + gothrow("all goroutines are asleep - deadlock!") +} + +func sysmon() { + // If we go two minutes without a garbage collection, force one to run. + forcegcperiod := int64(2 * 60 * 1e9) + + // If a heap span goes unused for 5 minutes after a garbage collection, + // we hand it back to the operating system. + scavengelimit := int64(5 * 60 * 1e9) + + if debug.scavenge > 0 { + // Scavenge-a-lot for testing. + forcegcperiod = 10 * 1e6 + scavengelimit = 20 * 1e6 + } + + lastscavenge := nanotime() + nscavenge := 0 + + // Make wake-up period small enough for the sampling to be correct. + maxsleep := forcegcperiod / 2 + if scavengelimit < forcegcperiod { + maxsleep = scavengelimit / 2 + } + + lasttrace := int64(0) + idle := 0 // how many cycles in succession we had not wokeup somebody + delay := uint32(0) + for { + if idle == 0 { // start with 20us sleep... + delay = 20 + } else if idle > 50 { // start doubling the sleep after 1ms... + delay *= 2 + } + if delay > 10*1000 { // up to 10ms + delay = 10 * 1000 + } + usleep(delay) + if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic + lock(&sched.lock) + if atomicload(&sched.gcwaiting) != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs) { + atomicstore(&sched.sysmonwait, 1) + unlock(&sched.lock) + notetsleep(&sched.sysmonnote, maxsleep) + lock(&sched.lock) + atomicstore(&sched.sysmonwait, 0) + noteclear(&sched.sysmonnote) + idle = 0 + delay = 20 + } + unlock(&sched.lock) + } + // poll network if not polled for more than 10ms + lastpoll := int64(atomicload64(&sched.lastpoll)) + now := nanotime() + unixnow := unixnanotime() + if lastpoll != 0 && lastpoll+10*1000*1000 < now { + cas64(&sched.lastpoll, uint64(lastpoll), uint64(now)) + gp := netpoll(false) // non-blocking - returns list of goroutines + if gp != nil { + // Need to decrement number of idle locked M's + // (pretending that one more is running) before injectglist. + // Otherwise it can lead to the following situation: + // injectglist grabs all P's but before it starts M's to run the P's, + // another M returns from syscall, finishes running its G, + // observes that there is no work to do and no other running M's + // and reports deadlock. + incidlelocked(-1) + injectglist(gp) + incidlelocked(1) + } + } + // retake P's blocked in syscalls + // and preempt long running G's + if retake(now) != 0 { + idle = 0 + } else { + idle++ + } + // check if we need to force a GC + lastgc := int64(atomicload64(&memstats.last_gc)) + if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 { + lock(&forcegc.lock) + forcegc.idle = 0 + forcegc.g.schedlink = nil + injectglist(forcegc.g) + unlock(&forcegc.lock) + } + // scavenge heap once in a while + if lastscavenge+scavengelimit/2 < now { + mHeap_Scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit)) + lastscavenge = now + nscavenge++ + } + if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace*1000000) <= now { + lasttrace = now + schedtrace(debug.scheddetail > 0) + } + } +} + +var pdesc [_MaxGomaxprocs]struct { + schedtick uint32 + schedwhen int64 + syscalltick uint32 + syscallwhen int64 +} + +func retake(now int64) uint32 { + n := 0 + for i := int32(0); i < gomaxprocs; i++ { + _p_ := allp[i] + if _p_ == nil { + continue + } + pd := &pdesc[i] + s := _p_.status + if s == _Psyscall { + // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). + t := int64(_p_.syscalltick) + if int64(pd.syscalltick) != t { + pd.syscalltick = uint32(t) + pd.syscallwhen = now + continue + } + // On the one hand we don't want to retake Ps if there is no other work to do, + // but on the other hand we want to retake them eventually + // because they can prevent the sysmon thread from deep sleep. + if _p_.runqhead == _p_.runqtail && atomicload(&sched.nmspinning)+atomicload(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now { + continue + } + // Need to decrement number of idle locked M's + // (pretending that one more is running) before the CAS. + // Otherwise the M from which we retake can exit the syscall, + // increment nmidle and report deadlock. + incidlelocked(-1) + if cas(&_p_.status, s, _Pidle) { + n++ + handoffp(_p_) + } + incidlelocked(1) + } else if s == _Prunning { + // Preempt G if it's running for more than 10ms. + t := int64(_p_.schedtick) + if int64(pd.schedtick) != t { + pd.schedtick = uint32(t) + pd.schedwhen = now + continue + } + if pd.schedwhen+10*1000*1000 > now { + continue + } + preemptone(_p_) + } + } + return uint32(n) +} + +// Tell all goroutines that they have been preempted and they should stop. +// This function is purely best-effort. It can fail to inform a goroutine if a +// processor just started running it. +// No locks need to be held. +// Returns true if preemption request was issued to at least one goroutine. +func preemptall() bool { + res := false + for i := int32(0); i < gomaxprocs; i++ { + _p_ := allp[i] + if _p_ == nil || _p_.status != _Prunning { + continue + } + if preemptone(_p_) { + res = true + } + } + return res +} + +// Tell the goroutine running on processor P to stop. +// This function is purely best-effort. It can incorrectly fail to inform the +// goroutine. It can send inform the wrong goroutine. Even if it informs the +// correct goroutine, that goroutine might ignore the request if it is +// simultaneously executing newstack. +// No lock needs to be held. +// Returns true if preemption request was issued. +// The actual preemption will happen at some point in the future +// and will be indicated by the gp->status no longer being +// Grunning +func preemptone(_p_ *p) bool { + mp := _p_.m + if mp == nil || mp == getg().m { + return false + } + gp := mp.curg + if gp == nil || gp == mp.g0 { + return false + } + + gp.preempt = true + + // Every call in a go routine checks for stack overflow by + // comparing the current stack pointer to gp->stackguard0. + // Setting gp->stackguard0 to StackPreempt folds + // preemption into the normal stack overflow check. + gp.stackguard0 = stackPreempt + return true +} + +var starttime int64 + +func schedtrace(detailed bool) { + now := nanotime() + if starttime == 0 { + starttime = now + } + + lock(&sched.lock) + print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize) + if detailed { + print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n") + } + // We must be careful while reading data from P's, M's and G's. + // Even if we hold schedlock, most data can be changed concurrently. + // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. + for i := int32(0); i < gomaxprocs; i++ { + _p_ := allp[i] + if _p_ == nil { + continue + } + mp := _p_.m + h := atomicload(&_p_.runqhead) + t := atomicload(&_p_.runqtail) + if detailed { + id := int32(-1) + if mp != nil { + id = mp.id + } + print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n") + } else { + // In non-detailed mode format lengths of per-P run queues as: + // [len1 len2 len3 len4] + print(" ") + if i == 0 { + print("[") + } + print(t - h) + if i == gomaxprocs-1 { + print("]\n") + } + } + } + + if !detailed { + unlock(&sched.lock) + return + } + + for mp := allm; mp != nil; mp = mp.alllink { + _p_ := mp.p + gp := mp.curg + lockedg := mp.lockedg + id1 := int32(-1) + if _p_ != nil { + id1 = _p_.id + } + id2 := int64(-1) + if gp != nil { + id2 = gp.goid + } + id3 := int64(-1) + if lockedg != nil { + id3 = lockedg.goid + } + print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " gcing=", mp.gcing, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n") + } + + lock(&allglock) + for gi := 0; gi < len(allgs); gi++ { + gp := allgs[gi] + mp := gp.m + lockedm := gp.lockedm + id1 := int32(-1) + if mp != nil { + id1 = mp.id + } + id2 := int32(-1) + if lockedm != nil { + id2 = lockedm.id + } + print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n") + } + unlock(&allglock) + unlock(&sched.lock) +} + +// Put mp on midle list. +// Sched must be locked. +func mput(mp *m) { + mp.schedlink = sched.midle + sched.midle = mp + sched.nmidle++ + checkdead() +} + +// Try to get an m from midle list. +// Sched must be locked. +func mget() *m { + mp := sched.midle + if mp != nil { + sched.midle = mp.schedlink + sched.nmidle-- + } + return mp +} + +// Put gp on the global runnable queue. +// Sched must be locked. +func globrunqput(gp *g) { + gp.schedlink = nil + if sched.runqtail != nil { + sched.runqtail.schedlink = gp + } else { + sched.runqhead = gp + } + sched.runqtail = gp + sched.runqsize++ +} + +// Put a batch of runnable goroutines on the global runnable queue. +// Sched must be locked. +func globrunqputbatch(ghead *g, gtail *g, n int32) { + gtail.schedlink = nil + if sched.runqtail != nil { + sched.runqtail.schedlink = ghead + } else { + sched.runqhead = ghead + } + sched.runqtail = gtail + sched.runqsize += n +} + +// Try get a batch of G's from the global runnable queue. +// Sched must be locked. +func globrunqget(_p_ *p, max int32) *g { + if sched.runqsize == 0 { + return nil + } + + n := sched.runqsize/gomaxprocs + 1 + if n > sched.runqsize { + n = sched.runqsize + } + if max > 0 && n > max { + n = max + } + if n > int32(len(_p_.runq))/2 { + n = int32(len(_p_.runq)) / 2 + } + + sched.runqsize -= n + if sched.runqsize == 0 { + sched.runqtail = nil + } + + gp := sched.runqhead + sched.runqhead = gp.schedlink + n-- + for ; n > 0; n-- { + gp1 := sched.runqhead + sched.runqhead = gp1.schedlink + runqput(_p_, gp1) + } + return gp +} + +// Put p to on _Pidle list. +// Sched must be locked. +func pidleput(_p_ *p) { + _p_.link = sched.pidle + sched.pidle = _p_ + xadd(&sched.npidle, 1) // TODO: fast atomic +} + +// Try get a p from _Pidle list. +// Sched must be locked. +func pidleget() *p { + _p_ := sched.pidle + if _p_ != nil { + sched.pidle = _p_.link + xadd(&sched.npidle, -1) // TODO: fast atomic + } + return _p_ +} + +// Try to put g on local runnable queue. +// If it's full, put onto global queue. +// Executed only by the owner P. +func runqput(_p_ *p, gp *g) { +retry: + h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers + t := _p_.runqtail + if t-h < uint32(len(_p_.runq)) { + _p_.runq[t%uint32(len(_p_.runq))] = gp + atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption + return + } + if runqputslow(_p_, gp, h, t) { + return + } + // the queue is not full, now the put above must suceed + goto retry +} + +// Put g and a batch of work from local runnable queue on global queue. +// Executed only by the owner P. +func runqputslow(_p_ *p, gp *g, h, t uint32) bool { + var batch [len(_p_.runq)/2 + 1]*g + + // First, grab a batch from local queue. + n := t - h + n = n / 2 + if n != uint32(len(_p_.runq)/2) { + gothrow("runqputslow: queue is not full") + } + for i := uint32(0); i < n; i++ { + batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))] + } + if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume + return false + } + batch[n] = gp + + // Link the goroutines. + for i := uint32(0); i < n; i++ { + batch[i].schedlink = batch[i+1] + } + + // Now put the batch on global queue. + lock(&sched.lock) + globrunqputbatch(batch[0], batch[n], int32(n+1)) + unlock(&sched.lock) + return true +} + +// Get g from local runnable queue. +// Executed only by the owner P. +func runqget(_p_ *p) *g { + for { + h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers + t := _p_.runqtail + if t == h { + return nil + } + gp := _p_.runq[h%uint32(len(_p_.runq))] + if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume + return gp + } + } +} + +// Grabs a batch of goroutines from local runnable queue. +// batch array must be of size len(p->runq)/2. Returns number of grabbed goroutines. +// Can be executed by any P. +func runqgrab(_p_ *p, batch []*g) uint32 { + for { + h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers + t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer + n := t - h + n = n - n/2 + if n == 0 { + return 0 + } + if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t + continue + } + for i := uint32(0); i < n; i++ { + batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))] + } + if cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume + return n + } + } +} + +// Steal half of elements from local runnable queue of p2 +// and put onto local runnable queue of p. +// Returns one of the stolen elements (or nil if failed). +func runqsteal(_p_, p2 *p) *g { + var batch [len(_p_.runq) / 2]*g + + n := runqgrab(p2, batch[:]) + if n == 0 { + return nil + } + n-- + gp := batch[n] + if n == 0 { + return gp + } + h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers + t := _p_.runqtail + if t-h+n >= uint32(len(_p_.runq)) { + gothrow("runqsteal: runq overflow") + } + for i := uint32(0); i < n; i++ { + _p_.runq[(t+i)%uint32(len(_p_.runq))] = batch[i] + } + atomicstore(&_p_.runqtail, t+n) // store-release, makes the item available for consumption + return gp +} + +func testSchedLocalQueue() { + _p_ := new(p) + gs := make([]g, len(_p_.runq)) + for i := 0; i < len(_p_.runq); i++ { + if runqget(_p_) != nil { + gothrow("runq is not empty initially") + } + for j := 0; j < i; j++ { + runqput(_p_, &gs[i]) + } + for j := 0; j < i; j++ { + if runqget(_p_) != &gs[i] { + print("bad element at iter ", i, "/", j, "\n") + gothrow("bad element") + } + } + if runqget(_p_) != nil { + gothrow("runq is not empty afterwards") + } + } +} + +func testSchedLocalQueueSteal() { + p1 := new(p) + p2 := new(p) + gs := make([]g, len(p1.runq)) + for i := 0; i < len(p1.runq); i++ { + for j := 0; j < i; j++ { + gs[j].sig = 0 + runqput(p1, &gs[j]) + } + gp := runqsteal(p2, p1) + s := 0 + if gp != nil { + s++ + gp.sig++ + } + for { + gp = runqget(p2) + if gp == nil { + break + } + s++ + gp.sig++ + } + for { + gp = runqget(p1) + if gp == nil { + break + } + gp.sig++ + } + for j := 0; j < i; j++ { + if gs[j].sig != 1 { + print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n") + gothrow("bad element") + } + } + if s != i/2 && s != i/2+1 { + print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n") + gothrow("bad steal") + } + } +} + +func setMaxThreads(in int) (out int) { + lock(&sched.lock) + out = int(sched.maxmcount) + sched.maxmcount = int32(in) + checkmcount() + unlock(&sched.lock) + return +} + +var goexperiment string = "GOEXPERIMENT" // TODO: defined in zaexperiment.h + +func haveexperiment(name string) bool { + x := goexperiment + for x != "" { + xname := "" + i := index(x, ",") + if i < 0 { + xname, x = x, "" + } else { + xname, x = x[:i], x[i+1:] + } + if xname == name { + return true + } + } + return false +} + +//go:nosplit +func sync_procPin() int { + _g_ := getg() + mp := _g_.m + + mp.locks++ + return int(mp.p.id) +} + +//go:nosplit +func sync_procUnpin() { + _g_ := getg() + _g_.m.locks-- +} diff --git a/src/runtime/race.c b/src/runtime/race.c deleted file mode 100644 index 5b0d11664..000000000 --- a/src/runtime/race.c +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Implementation of the race detector API. -// +build race - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "race.h" -#include "type.h" -#include "typekind.h" -#include "textflag.h" - -// Race runtime functions called via runtime·racecall. -void __tsan_init(void); -void __tsan_fini(void); -void __tsan_map_shadow(void); -void __tsan_finalizer_goroutine(void); -void __tsan_go_start(void); -void __tsan_go_end(void); -void __tsan_malloc(void); -void __tsan_acquire(void); -void __tsan_release(void); -void __tsan_release_merge(void); -void __tsan_go_ignore_sync_begin(void); -void __tsan_go_ignore_sync_end(void); - -// Mimic what cmd/cgo would do. -#pragma cgo_import_static __tsan_init -#pragma cgo_import_static __tsan_fini -#pragma cgo_import_static __tsan_map_shadow -#pragma cgo_import_static __tsan_finalizer_goroutine -#pragma cgo_import_static __tsan_go_start -#pragma cgo_import_static __tsan_go_end -#pragma cgo_import_static __tsan_malloc -#pragma cgo_import_static __tsan_acquire -#pragma cgo_import_static __tsan_release -#pragma cgo_import_static __tsan_release_merge -#pragma cgo_import_static __tsan_go_ignore_sync_begin -#pragma cgo_import_static __tsan_go_ignore_sync_end - -// These are called from race_amd64.s. -#pragma cgo_import_static __tsan_read -#pragma cgo_import_static __tsan_read_pc -#pragma cgo_import_static __tsan_read_range -#pragma cgo_import_static __tsan_write -#pragma cgo_import_static __tsan_write_pc -#pragma cgo_import_static __tsan_write_range -#pragma cgo_import_static __tsan_func_enter -#pragma cgo_import_static __tsan_func_exit - -#pragma cgo_import_static __tsan_go_atomic32_load -#pragma cgo_import_static __tsan_go_atomic64_load -#pragma cgo_import_static __tsan_go_atomic32_store -#pragma cgo_import_static __tsan_go_atomic64_store -#pragma cgo_import_static __tsan_go_atomic32_exchange -#pragma cgo_import_static __tsan_go_atomic64_exchange -#pragma cgo_import_static __tsan_go_atomic32_fetch_add -#pragma cgo_import_static __tsan_go_atomic64_fetch_add -#pragma cgo_import_static __tsan_go_atomic32_compare_exchange -#pragma cgo_import_static __tsan_go_atomic64_compare_exchange - -extern byte runtime·noptrdata[]; -extern byte runtime·enoptrdata[]; -extern byte runtime·data[]; -extern byte runtime·edata[]; -extern byte runtime·bss[]; -extern byte runtime·ebss[]; -extern byte runtime·noptrbss[]; -extern byte runtime·enoptrbss[]; - -// start/end of global data (data+bss). -uintptr runtime·racedatastart; -uintptr runtime·racedataend; -// start/end of heap for race_amd64.s -uintptr runtime·racearenastart; -uintptr runtime·racearenaend; - -void runtime·racefuncenter(void *callpc); -void runtime·racefuncexit(void); -void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc); -void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc); -void runtime·racesymbolizethunk(void*); - -// racecall allows calling an arbitrary function f from C race runtime -// with up to 4 uintptr arguments. -void runtime·racecall(void(*f)(void), ...); - -// checks if the address has shadow (i.e. heap or data/bss) -#pragma textflag NOSPLIT -static bool -isvalidaddr(uintptr addr) -{ - if(addr >= runtime·racearenastart && addr < runtime·racearenaend) - return true; - if(addr >= runtime·racedatastart && addr < runtime·racedataend) - return true; - return false; -} - -#pragma textflag NOSPLIT -uintptr -runtime·raceinit(void) -{ - uintptr racectx, start, end, size; - - // cgo is required to initialize libc, which is used by race runtime - if(!runtime·iscgo) - runtime·throw("raceinit: race build must use cgo"); - runtime·racecall(__tsan_init, &racectx, runtime·racesymbolizethunk); - // Round data segment to page boundaries, because it's used in mmap(). - // The relevant sections are noptrdata, data, bss, noptrbss. - // In external linking mode, there may be other non-Go data mixed in, - // and the sections may even occur out of order. - // Work out a conservative range of addresses. - start = ~(uintptr)0; - end = 0; - if(start > (uintptr)runtime·noptrdata) - start = (uintptr)runtime·noptrdata; - if(start > (uintptr)runtime·data) - start = (uintptr)runtime·data; - if(start > (uintptr)runtime·noptrbss) - start = (uintptr)runtime·noptrbss; - if(start > (uintptr)runtime·bss) - start = (uintptr)runtime·bss; - if(end < (uintptr)runtime·enoptrdata) - end = (uintptr)runtime·enoptrdata; - if(end < (uintptr)runtime·edata) - end = (uintptr)runtime·edata; - if(end < (uintptr)runtime·enoptrbss) - end = (uintptr)runtime·enoptrbss; - if(end < (uintptr)runtime·ebss) - end = (uintptr)runtime·ebss; - start = start & ~(PageSize-1); - size = ROUND(end - start, PageSize); - runtime·racecall(__tsan_map_shadow, start, size); - runtime·racedatastart = start; - runtime·racedataend = start + size; - return racectx; -} - -#pragma textflag NOSPLIT -void -runtime·racefini(void) -{ - runtime·racecall(__tsan_fini); -} - -#pragma textflag NOSPLIT -void -runtime·racemapshadow(void *addr, uintptr size) -{ - if(runtime·racearenastart == 0) - runtime·racearenastart = (uintptr)addr; - if(runtime·racearenaend < (uintptr)addr+size) - runtime·racearenaend = (uintptr)addr+size; - runtime·racecall(__tsan_map_shadow, addr, size); -} - -#pragma textflag NOSPLIT -void -runtime·racemalloc(void *p, uintptr sz) -{ - runtime·racecall(__tsan_malloc, p, sz); -} - -#pragma textflag NOSPLIT -uintptr -runtime·racegostart(void *pc) -{ - uintptr racectx; - G *spawng; - - if(g->m->curg != nil) - spawng = g->m->curg; - else - spawng = g; - - runtime·racecall(__tsan_go_start, spawng->racectx, &racectx, pc); - return racectx; -} - -#pragma textflag NOSPLIT -void -runtime·racegoend(void) -{ - runtime·racecall(__tsan_go_end, g->racectx); -} - -#pragma textflag NOSPLIT -void -runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc) -{ - if(g != g->m->curg) { - // The call is coming from manual instrumentation of Go code running on g0/gsignal. - // Not interesting. - return; - } - if(callpc != nil) - runtime·racefuncenter(callpc); - runtime·racewriterangepc1(addr, sz, pc); - if(callpc != nil) - runtime·racefuncexit(); -} - -#pragma textflag NOSPLIT -void -runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc) -{ - if(g != g->m->curg) { - // The call is coming from manual instrumentation of Go code running on g0/gsignal. - // Not interesting. - return; - } - if(callpc != nil) - runtime·racefuncenter(callpc); - runtime·racereadrangepc1(addr, sz, pc); - if(callpc != nil) - runtime·racefuncexit(); -} - -#pragma textflag NOSPLIT -void -runtime·racewriteobjectpc(void *addr, Type *t, void *callpc, void *pc) -{ - uint8 kind; - - kind = t->kind & KindMask; - if(kind == KindArray || kind == KindStruct) - runtime·racewriterangepc(addr, t->size, callpc, pc); - else - runtime·racewritepc(addr, callpc, pc); -} - -#pragma textflag NOSPLIT -void -runtime·racereadobjectpc(void *addr, Type *t, void *callpc, void *pc) -{ - uint8 kind; - - kind = t->kind & KindMask; - if(kind == KindArray || kind == KindStruct) - runtime·racereadrangepc(addr, t->size, callpc, pc); - else - runtime·racereadpc(addr, callpc, pc); -} - -#pragma textflag NOSPLIT -void -runtime·raceacquire(void *addr) -{ - runtime·raceacquireg(g, addr); -} - -#pragma textflag NOSPLIT -void -runtime·raceacquireg(G *gp, void *addr) -{ - if(g->raceignore || !isvalidaddr((uintptr)addr)) - return; - runtime·racecall(__tsan_acquire, gp->racectx, addr); -} - -#pragma textflag NOSPLIT -void -runtime·racerelease(void *addr) -{ - if(g->raceignore || !isvalidaddr((uintptr)addr)) - return; - runtime·racereleaseg(g, addr); -} - -#pragma textflag NOSPLIT -void -runtime·racereleaseg(G *gp, void *addr) -{ - if(g->raceignore || !isvalidaddr((uintptr)addr)) - return; - runtime·racecall(__tsan_release, gp->racectx, addr); -} - -#pragma textflag NOSPLIT -void -runtime·racereleasemerge(void *addr) -{ - runtime·racereleasemergeg(g, addr); -} - -#pragma textflag NOSPLIT -void -runtime·racereleasemergeg(G *gp, void *addr) -{ - if(g->raceignore || !isvalidaddr((uintptr)addr)) - return; - runtime·racecall(__tsan_release_merge, gp->racectx, addr); -} - -#pragma textflag NOSPLIT -void -runtime·racefingo(void) -{ - runtime·racecall(__tsan_finalizer_goroutine, g->racectx); -} - -// func RaceAcquire(addr unsafe.Pointer) -#pragma textflag NOSPLIT -void -runtime·RaceAcquire(void *addr) -{ - runtime·raceacquire(addr); -} - -// func RaceRelease(addr unsafe.Pointer) -#pragma textflag NOSPLIT -void -runtime·RaceRelease(void *addr) -{ - runtime·racerelease(addr); -} - -// func RaceReleaseMerge(addr unsafe.Pointer) -#pragma textflag NOSPLIT -void -runtime·RaceReleaseMerge(void *addr) -{ - runtime·racereleasemerge(addr); -} - -// func RaceDisable() -#pragma textflag NOSPLIT -void -runtime·RaceDisable(void) -{ - if(g->raceignore++ == 0) - runtime·racecall(__tsan_go_ignore_sync_begin, g->racectx); -} - -// func RaceEnable() -#pragma textflag NOSPLIT -void -runtime·RaceEnable(void) -{ - if(--g->raceignore == 0) - runtime·racecall(__tsan_go_ignore_sync_end, g->racectx); -} diff --git a/src/runtime/race.go b/src/runtime/race.go index bb0ee6df6..649cd7295 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -12,18 +12,6 @@ import ( "unsafe" ) -func racefini() - -// RaceDisable disables handling of race events in the current goroutine. -func RaceDisable() - -// RaceEnable re-enables handling of race events in the current goroutine. -func RaceEnable() - -func RaceAcquire(addr unsafe.Pointer) -func RaceRelease(addr unsafe.Pointer) -func RaceReleaseMerge(addr unsafe.Pointer) - func RaceRead(addr unsafe.Pointer) func RaceWrite(addr unsafe.Pointer) func RaceReadRange(addr unsafe.Pointer, len int) @@ -67,32 +55,6 @@ func racereadpc(addr unsafe.Pointer, callpc, pc uintptr) //go:noescape func racewritepc(addr unsafe.Pointer, callpc, pc uintptr) -//go:noescape -func racereadrangepc(addr unsafe.Pointer, len uintptr, callpc, pc uintptr) - -//go:noescape -func racewriterangepc(addr unsafe.Pointer, len uintptr, callpc, pc uintptr) - -//go:noescape -func raceacquire(addr unsafe.Pointer) - -//go:noescape -func racerelease(addr unsafe.Pointer) - -//go:noescape -func raceacquireg(gp *g, addr unsafe.Pointer) - -//go:noescape -func racereleaseg(gp *g, addr unsafe.Pointer) - -func racefingo() - -//go:noescape -func racemalloc(p unsafe.Pointer, size uintptr) - -//go:noescape -func racereleasemerge(addr unsafe.Pointer) - type symbolizeContext struct { pc uintptr fn *byte @@ -118,8 +80,8 @@ func racesymbolize(ctx *symbolizeContext) { } ctx.fn = funcname(f) - var file string - ctx.line = uintptr(funcline(f, ctx.pc, &file)) + file, line := funcline(f, ctx.pc) + ctx.line = uintptr(line) ctx.file = &bytes(file)[0] // assume NUL-terminated ctx.off = ctx.pc - f.entry ctx.res = 1 diff --git a/src/runtime/race.h b/src/runtime/race.h deleted file mode 100644 index fee31e09f..000000000 --- a/src/runtime/race.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Definitions related to data race detection. - -#ifdef RACE -enum { raceenabled = 1 }; -#else -enum { raceenabled = 0 }; -#endif - -// Initialize race detection subsystem. -uintptr runtime·raceinit(void); -// Finalize race detection subsystem, does not return. -void runtime·racefini(void); - -void runtime·racemapshadow(void *addr, uintptr size); -void runtime·racemalloc(void *p, uintptr sz); -uintptr runtime·racegostart(void *pc); -void runtime·racegoend(void); -void runtime·racewritepc(void *addr, void *callpc, void *pc); -void runtime·racereadpc(void *addr, void *callpc, void *pc); -void runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc); -void runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc); -void runtime·racereadobjectpc(void *addr, Type *t, void *callpc, void *pc); -void runtime·racewriteobjectpc(void *addr, Type *t, void *callpc, void *pc); -void runtime·racefingo(void); -void runtime·raceacquire(void *addr); -void runtime·raceacquireg(G *gp, void *addr); -void runtime·racerelease(void *addr); -void runtime·racereleaseg(G *gp, void *addr); -void runtime·racereleasemerge(void *addr); -void runtime·racereleasemergeg(G *gp, void *addr); diff --git a/src/runtime/race0.go b/src/runtime/race0.go index 5d90cc859..dadb6083f 100644 --- a/src/runtime/race0.go +++ b/src/runtime/race0.go @@ -18,7 +18,7 @@ const raceenabled = false func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { gothrow("race") } func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { gothrow("race") } -func raceinit() { gothrow("race") } +func raceinit() uintptr { gothrow("race"); return 0 } func racefini() { gothrow("race") } func racemapshadow(addr unsafe.Pointer, size uintptr) { gothrow("race") } func racewritepc(addr unsafe.Pointer, callerpc, pc uintptr) { gothrow("race") } diff --git a/src/runtime/race1.go b/src/runtime/race1.go new file mode 100644 index 000000000..2ec2bee65 --- /dev/null +++ b/src/runtime/race1.go @@ -0,0 +1,335 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implementation of the race detector API. +// +build race + +package runtime + +import "unsafe" + +// Race runtime functions called via runtime·racecall. +//go:linkname __tsan_init __tsan_init +var __tsan_init byte + +//go:linkname __tsan_fini __tsan_fini +var __tsan_fini byte + +//go:linkname __tsan_map_shadow __tsan_map_shadow +var __tsan_map_shadow byte + +//go:linkname __tsan_finalizer_goroutine __tsan_finalizer_goroutine +var __tsan_finalizer_goroutine byte + +//go:linkname __tsan_go_start __tsan_go_start +var __tsan_go_start byte + +//go:linkname __tsan_go_end __tsan_go_end +var __tsan_go_end byte + +//go:linkname __tsan_malloc __tsan_malloc +var __tsan_malloc byte + +//go:linkname __tsan_acquire __tsan_acquire +var __tsan_acquire byte + +//go:linkname __tsan_release __tsan_release +var __tsan_release byte + +//go:linkname __tsan_release_merge __tsan_release_merge +var __tsan_release_merge byte + +//go:linkname __tsan_go_ignore_sync_begin __tsan_go_ignore_sync_begin +var __tsan_go_ignore_sync_begin byte + +//go:linkname __tsan_go_ignore_sync_end __tsan_go_ignore_sync_end +var __tsan_go_ignore_sync_end byte + +// Mimic what cmd/cgo would do. +//go:cgo_import_static __tsan_init +//go:cgo_import_static __tsan_fini +//go:cgo_import_static __tsan_map_shadow +//go:cgo_import_static __tsan_finalizer_goroutine +//go:cgo_import_static __tsan_go_start +//go:cgo_import_static __tsan_go_end +//go:cgo_import_static __tsan_malloc +//go:cgo_import_static __tsan_acquire +//go:cgo_import_static __tsan_release +//go:cgo_import_static __tsan_release_merge +//go:cgo_import_static __tsan_go_ignore_sync_begin +//go:cgo_import_static __tsan_go_ignore_sync_end + +// These are called from race_amd64.s. +//go:cgo_import_static __tsan_read +//go:cgo_import_static __tsan_read_pc +//go:cgo_import_static __tsan_read_range +//go:cgo_import_static __tsan_write +//go:cgo_import_static __tsan_write_pc +//go:cgo_import_static __tsan_write_range +//go:cgo_import_static __tsan_func_enter +//go:cgo_import_static __tsan_func_exit + +//go:cgo_import_static __tsan_go_atomic32_load +//go:cgo_import_static __tsan_go_atomic64_load +//go:cgo_import_static __tsan_go_atomic32_store +//go:cgo_import_static __tsan_go_atomic64_store +//go:cgo_import_static __tsan_go_atomic32_exchange +//go:cgo_import_static __tsan_go_atomic64_exchange +//go:cgo_import_static __tsan_go_atomic32_fetch_add +//go:cgo_import_static __tsan_go_atomic64_fetch_add +//go:cgo_import_static __tsan_go_atomic32_compare_exchange +//go:cgo_import_static __tsan_go_atomic64_compare_exchange + +// start/end of global data (data+bss). +var racedatastart uintptr +var racedataend uintptr + +// start/end of heap for race_amd64.s +var racearenastart uintptr +var racearenaend uintptr + +func racefuncenter(uintptr) +func racefuncexit() +func racereadrangepc1(uintptr, uintptr, uintptr) +func racewriterangepc1(uintptr, uintptr, uintptr) +func racesymbolizethunk(uintptr) + +// racecall allows calling an arbitrary function f from C race runtime +// with up to 4 uintptr arguments. +func racecall(*byte, uintptr, uintptr, uintptr, uintptr) + +// checks if the address has shadow (i.e. heap or data/bss) +//go:nosplit +func isvalidaddr(addr unsafe.Pointer) bool { + return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend || + racedatastart <= uintptr(addr) && uintptr(addr) < racedataend +} + +//go:nosplit +func raceinit() uintptr { + // cgo is required to initialize libc, which is used by race runtime + if !iscgo { + gothrow("raceinit: race build must use cgo") + } + + var racectx uintptr + racecall(&__tsan_init, uintptr(unsafe.Pointer(&racectx)), funcPC(racesymbolizethunk), 0, 0) + + // Round data segment to page boundaries, because it's used in mmap(). + start := ^uintptr(0) + end := uintptr(0) + if start > uintptr(unsafe.Pointer(&noptrdata)) { + start = uintptr(unsafe.Pointer(&noptrdata)) + } + if start > uintptr(unsafe.Pointer(&data)) { + start = uintptr(unsafe.Pointer(&data)) + } + if start > uintptr(unsafe.Pointer(&noptrbss)) { + start = uintptr(unsafe.Pointer(&noptrbss)) + } + if start > uintptr(unsafe.Pointer(&bss)) { + start = uintptr(unsafe.Pointer(&bss)) + } + if end < uintptr(unsafe.Pointer(&enoptrdata)) { + end = uintptr(unsafe.Pointer(&enoptrdata)) + } + if end < uintptr(unsafe.Pointer(&edata)) { + end = uintptr(unsafe.Pointer(&edata)) + } + if end < uintptr(unsafe.Pointer(&enoptrbss)) { + end = uintptr(unsafe.Pointer(&enoptrbss)) + } + if end < uintptr(unsafe.Pointer(&ebss)) { + end = uintptr(unsafe.Pointer(&ebss)) + } + size := round(end-start, _PageSize) + racecall(&__tsan_map_shadow, start, size, 0, 0) + racedatastart = start + racedataend = start + size + + return racectx +} + +//go:nosplit +func racefini() { + racecall(&__tsan_fini, 0, 0, 0, 0) +} + +//go:nosplit +func racemapshadow(addr unsafe.Pointer, size uintptr) { + if racearenastart == 0 { + racearenastart = uintptr(addr) + } + if racearenaend < uintptr(addr)+size { + racearenaend = uintptr(addr) + size + } + racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0) +} + +//go:nosplit +func racemalloc(p unsafe.Pointer, sz uintptr) { + racecall(&__tsan_malloc, uintptr(p), sz, 0, 0) +} + +//go:nosplit +func racegostart(pc uintptr) uintptr { + _g_ := getg() + var spawng *g + if _g_.m.curg != nil { + spawng = _g_.m.curg + } else { + spawng = _g_ + } + + var racectx uintptr + racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0) + return racectx +} + +//go:nosplit +func racegoend() { + racecall(&__tsan_go_end, getg().racectx, 0, 0, 0) +} + +//go:nosplit +func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) { + _g_ := getg() + if _g_ != _g_.m.curg { + // The call is coming from manual instrumentation of Go code running on g0/gsignal. + // Not interesting. + return + } + if callpc != 0 { + racefuncenter(callpc) + } + racewriterangepc1(uintptr(addr), sz, pc) + if callpc != 0 { + racefuncexit() + } +} + +//go:nosplit +func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) { + _g_ := getg() + if _g_ != _g_.m.curg { + // The call is coming from manual instrumentation of Go code running on g0/gsignal. + // Not interesting. + return + } + if callpc != 0 { + racefuncenter(callpc) + } + racereadrangepc1(uintptr(addr), sz, pc) + if callpc != 0 { + racefuncexit() + } +} + +//go:nosplit +func racewriteobjectpc(addr unsafe.Pointer, t *_type, callpc, pc uintptr) { + kind := t.kind & _KindMask + if kind == _KindArray || kind == _KindStruct { + racewriterangepc(addr, t.size, callpc, pc) + } else { + racewritepc(addr, callpc, pc) + } +} + +//go:nosplit +func racereadobjectpc(addr unsafe.Pointer, t *_type, callpc, pc uintptr) { + kind := t.kind & _KindMask + if kind == _KindArray || kind == _KindStruct { + racereadrangepc(addr, t.size, callpc, pc) + } else { + racereadpc(addr, callpc, pc) + } +} + +//go:nosplit +func raceacquire(addr unsafe.Pointer) { + raceacquireg(getg(), addr) +} + +//go:nosplit +func raceacquireg(gp *g, addr unsafe.Pointer) { + if getg().raceignore != 0 || !isvalidaddr(addr) { + return + } + racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0) +} + +//go:nosplit +func racerelease(addr unsafe.Pointer) { + _g_ := getg() + if _g_.raceignore != 0 || !isvalidaddr(addr) { + return + } + racereleaseg(_g_, addr) +} + +//go:nosplit +func racereleaseg(gp *g, addr unsafe.Pointer) { + if getg().raceignore != 0 || !isvalidaddr(addr) { + return + } + racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0) +} + +//go:nosplit +func racereleasemerge(addr unsafe.Pointer) { + racereleasemergeg(getg(), addr) +} + +//go:nosplit +func racereleasemergeg(gp *g, addr unsafe.Pointer) { + if getg().raceignore != 0 || !isvalidaddr(addr) { + return + } + racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0) +} + +//go:nosplit +func racefingo() { + racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0) +} + +//go:nosplit + +func RaceAcquire(addr unsafe.Pointer) { + raceacquire(addr) +} + +//go:nosplit + +func RaceRelease(addr unsafe.Pointer) { + racerelease(addr) +} + +//go:nosplit + +func RaceReleaseMerge(addr unsafe.Pointer) { + racereleasemerge(addr) +} + +//go:nosplit + +// RaceEnable re-enables handling of race events in the current goroutine. +func RaceDisable() { + _g_ := getg() + if _g_.raceignore == 0 { + racecall(&__tsan_go_ignore_sync_begin, _g_.racectx, 0, 0, 0) + } + _g_.raceignore++ +} + +//go:nosplit + +// RaceDisable disables handling of race events in the current goroutine. +func RaceEnable() { + _g_ := getg() + _g_.raceignore-- + if _g_.raceignore == 0 { + racecall(&__tsan_go_ignore_sync_end, _g_.racectx, 0, 0, 0) + } +} diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index a96d9de12..d54d9798f 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -4,7 +4,8 @@ // +build race -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -140,18 +141,18 @@ TEXT racecalladdr<>(SB), NOSPLIT, $0-0 MOVQ g_racectx(R14), RARG0 // goroutine context // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend). CMPQ RARG1, runtime·racearenastart(SB) - JB racecalladdr_data + JB data CMPQ RARG1, runtime·racearenaend(SB) - JB racecalladdr_call -racecalladdr_data: + JB call +data: CMPQ RARG1, runtime·racedatastart(SB) - JB racecalladdr_ret + JB ret CMPQ RARG1, runtime·racedataend(SB) - JAE racecalladdr_ret -racecalladdr_call: + JAE ret +call: MOVQ AX, AX // w/o this 6a miscompiles this function JMP racecall<>(SB) -racecalladdr_ret: +ret: RET // func runtime·racefuncenter(pc uintptr) @@ -366,9 +367,9 @@ TEXT racecall<>(SB), NOSPLIT, $0-0 MOVQ SP, R12 // callee-saved, preserved across the CALL MOVQ m_g0(R13), R10 CMPQ R10, R14 - JE racecall_cont // already on g0 + JE call // already on g0 MOVQ (g_sched+gobuf_sp)(R10), SP -racecall_cont: +call: ANDQ $~15, SP // alignment for gcc ABI CALL AX MOVQ R12, SP diff --git a/src/runtime/rdebug.go b/src/runtime/rdebug.go index e5e691122..f2766d793 100644 --- a/src/runtime/rdebug.go +++ b/src/runtime/rdebug.go @@ -10,15 +10,6 @@ func setMaxStack(in int) (out int) { return out } -func setGCPercent(in int32) (out int32) { - mp := acquirem() - mp.scalararg[0] = uintptr(int(in)) - onM(setgcpercent_m) - out = int32(int(mp.scalararg[0])) - releasem(mp) - return out -} - func setPanicOnFault(new bool) (old bool) { mp := acquirem() old = mp.curg.paniconfault @@ -26,12 +17,3 @@ func setPanicOnFault(new bool) (old bool) { releasem(mp) return old } - -func setMaxThreads(in int) (out int) { - mp := acquirem() - mp.scalararg[0] = uintptr(in) - onM(setmaxthreads_m) - out = int(mp.scalararg[0]) - releasem(mp) - return out -} diff --git a/src/runtime/rt0_linux_power64.s b/src/runtime/rt0_linux_power64.s new file mode 100644 index 000000000..970b6a673 --- /dev/null +++ b/src/runtime/rt0_linux_power64.s @@ -0,0 +1,17 @@ +#include "textflag.h" + +// actually a function descriptor for _main<>(SB) +TEXT _rt0_power64_linux(SB),NOSPLIT,$0 + DWORD $_main<>(SB) + DWORD $0 + DWORD $0 + +TEXT _main<>(SB),NOSPLIT,$-8 + MOVD 0(R1), R3 // argc + ADD $8, R1, R4 // argv + BR main(SB) + +TEXT main(SB),NOSPLIT,$-8 + MOVD $runtime·rt0_go(SB), R31 + MOVD R31, CTR + BR (CTR) diff --git a/src/runtime/rt0_linux_power64le.s b/src/runtime/rt0_linux_power64le.s new file mode 100644 index 000000000..85ce84733 --- /dev/null +++ b/src/runtime/rt0_linux_power64le.s @@ -0,0 +1,14 @@ +#include "textflag.h" + +TEXT _rt0_power64le_linux(SB),NOSPLIT,$0 + BR _main<>(SB) + +TEXT _main<>(SB),NOSPLIT,$-8 + MOVD 0(R1), R3 // argc + ADD $8, R1, R4 // argv + BR main(SB) + +TEXT main(SB),NOSPLIT,$-8 + MOVD $runtime·rt0_go(SB), R31 + MOVD R31, CTR + BR (CTR) diff --git a/src/runtime/rt0_windows_amd64.s b/src/runtime/rt0_windows_amd64.s index 197f52e11..df956ba36 100644 --- a/src/runtime/rt0_windows_amd64.s +++ b/src/runtime/rt0_windows_amd64.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT _rt0_amd64_windows(SB),NOSPLIT,$-8 diff --git a/src/runtime/runtime.c b/src/runtime/runtime.c deleted file mode 100644 index c823691ec..000000000 --- a/src/runtime/runtime.c +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "stack.h" -#include "arch_GOARCH.h" -#include "textflag.h" -#include "malloc.h" - -// Keep a cached value to make gotraceback fast, -// since we call it on every call to gentraceback. -// The cached value is a uint32 in which the low bit -// is the "crash" setting and the top 31 bits are the -// gotraceback value. -static uint32 traceback_cache = 2<<1; - -// The GOTRACEBACK environment variable controls the -// behavior of a Go program that is crashing and exiting. -// GOTRACEBACK=0 suppress all tracebacks -// GOTRACEBACK=1 default behavior - show tracebacks but exclude runtime frames -// GOTRACEBACK=2 show tracebacks including runtime frames -// GOTRACEBACK=crash show tracebacks including runtime frames, then crash (core dump etc) -#pragma textflag NOSPLIT -int32 -runtime·gotraceback(bool *crash) -{ - if(crash != nil) - *crash = false; - if(g->m->traceback != 0) - return g->m->traceback; - if(crash != nil) - *crash = traceback_cache&1; - return traceback_cache>>1; -} - -int32 -runtime·mcmp(byte *s1, byte *s2, uintptr n) -{ - uintptr i; - byte c1, c2; - - for(i=0; i<n; i++) { - c1 = s1[i]; - c2 = s2[i]; - if(c1 < c2) - return -1; - if(c1 > c2) - return +1; - } - return 0; -} - - -byte* -runtime·mchr(byte *p, byte c, byte *ep) -{ - for(; p < ep; p++) - if(*p == c) - return p; - return nil; -} - -static int32 argc; - -#pragma dataflag NOPTR /* argv not a heap pointer */ -static uint8** argv; - -extern Slice runtime·argslice; -extern Slice runtime·envs; - -void (*runtime·sysargs)(int32, uint8**); - -void -runtime·args(int32 c, uint8 **v) -{ - argc = c; - argv = v; - if(runtime·sysargs != nil) - runtime·sysargs(c, v); -} - -int32 runtime·isplan9; -int32 runtime·issolaris; -int32 runtime·iswindows; - -// Information about what cpu features are available. -// Set on startup in asm_{x86/amd64}.s. -uint32 runtime·cpuid_ecx; -uint32 runtime·cpuid_edx; - -void -runtime·goargs(void) -{ - String *s; - int32 i; - - // for windows implementation see "os" package - if(Windows) - return; - - runtime·argslice = runtime·makeStringSlice(argc); - s = (String*)runtime·argslice.array; - for(i=0; i<argc; i++) - s[i] = runtime·gostringnocopy(argv[i]); -} - -void -runtime·goenvs_unix(void) -{ - String *s; - int32 i, n; - - for(n=0; argv[argc+1+n] != 0; n++) - ; - - runtime·envs = runtime·makeStringSlice(n); - s = (String*)runtime·envs.array; - for(i=0; i<n; i++) - s[i] = runtime·gostringnocopy(argv[argc+1+i]); -} - -#pragma textflag NOSPLIT -Slice -runtime·environ() -{ - return runtime·envs; -} - -int32 -runtime·atoi(byte *p) -{ - int32 n; - - n = 0; - while('0' <= *p && *p <= '9') - n = n*10 + *p++ - '0'; - return n; -} - -static void -TestAtomic64(void) -{ - uint64 z64, x64; - - z64 = 42; - x64 = 0; - PREFETCH(&z64); - if(runtime·cas64(&z64, x64, 1)) - runtime·throw("cas64 failed"); - if(x64 != 0) - runtime·throw("cas64 failed"); - x64 = 42; - if(!runtime·cas64(&z64, x64, 1)) - runtime·throw("cas64 failed"); - if(x64 != 42 || z64 != 1) - runtime·throw("cas64 failed"); - if(runtime·atomicload64(&z64) != 1) - runtime·throw("load64 failed"); - runtime·atomicstore64(&z64, (1ull<<40)+1); - if(runtime·atomicload64(&z64) != (1ull<<40)+1) - runtime·throw("store64 failed"); - if(runtime·xadd64(&z64, (1ull<<40)+1) != (2ull<<40)+2) - runtime·throw("xadd64 failed"); - if(runtime·atomicload64(&z64) != (2ull<<40)+2) - runtime·throw("xadd64 failed"); - if(runtime·xchg64(&z64, (3ull<<40)+3) != (2ull<<40)+2) - runtime·throw("xchg64 failed"); - if(runtime·atomicload64(&z64) != (3ull<<40)+3) - runtime·throw("xchg64 failed"); -} - -void -runtime·check(void) -{ - int8 a; - uint8 b; - int16 c; - uint16 d; - int32 e; - uint32 f; - int64 g; - uint64 h; - float32 i, i1; - float64 j, j1; - byte *k, *k1; - uint16* l; - struct x1 { - byte x; - }; - struct y1 { - struct x1 x1; - byte y; - }; - - if(sizeof(a) != 1) runtime·throw("bad a"); - if(sizeof(b) != 1) runtime·throw("bad b"); - if(sizeof(c) != 2) runtime·throw("bad c"); - if(sizeof(d) != 2) runtime·throw("bad d"); - if(sizeof(e) != 4) runtime·throw("bad e"); - if(sizeof(f) != 4) runtime·throw("bad f"); - if(sizeof(g) != 8) runtime·throw("bad g"); - if(sizeof(h) != 8) runtime·throw("bad h"); - if(sizeof(i) != 4) runtime·throw("bad i"); - if(sizeof(j) != 8) runtime·throw("bad j"); - if(sizeof(k) != sizeof(uintptr)) runtime·throw("bad k"); - if(sizeof(l) != sizeof(uintptr)) runtime·throw("bad l"); - if(sizeof(struct x1) != 1) runtime·throw("bad sizeof x1"); - if(offsetof(struct y1, y) != 1) runtime·throw("bad offsetof y1.y"); - if(sizeof(struct y1) != 2) runtime·throw("bad sizeof y1"); - - if(runtime·timediv(12345LL*1000000000+54321, 1000000000, &e) != 12345 || e != 54321) - runtime·throw("bad timediv"); - - uint32 z; - z = 1; - if(!runtime·cas(&z, 1, 2)) - runtime·throw("cas1"); - if(z != 2) - runtime·throw("cas2"); - - z = 4; - if(runtime·cas(&z, 5, 6)) - runtime·throw("cas3"); - if(z != 4) - runtime·throw("cas4"); - - k = (byte*)0xfedcb123; - if(sizeof(void*) == 8) - k = (byte*)((uintptr)k<<10); - if(runtime·casp((void**)&k, nil, nil)) - runtime·throw("casp1"); - k1 = k+1; - if(!runtime·casp((void**)&k, k, k1)) - runtime·throw("casp2"); - if(k != k1) - runtime·throw("casp3"); - - *(uint64*)&j = ~0ULL; - if(j == j) - runtime·throw("float64nan"); - if(!(j != j)) - runtime·throw("float64nan1"); - - *(uint64*)&j1 = ~1ULL; - if(j == j1) - runtime·throw("float64nan2"); - if(!(j != j1)) - runtime·throw("float64nan3"); - - *(uint32*)&i = ~0UL; - if(i == i) - runtime·throw("float32nan"); - if(!(i != i)) - runtime·throw("float32nan1"); - - *(uint32*)&i1 = ~1UL; - if(i == i1) - runtime·throw("float32nan2"); - if(!(i != i1)) - runtime·throw("float32nan3"); - - TestAtomic64(); - - if(FixedStack != runtime·round2(FixedStack)) - runtime·throw("FixedStack is not power-of-2"); -} - -#pragma dataflag NOPTR -DebugVars runtime·debug; - -typedef struct DbgVar DbgVar; -struct DbgVar -{ - int8* name; - int32* value; -}; - -// Do we report invalid pointers found during stack or heap scans? -int32 runtime·invalidptr = 1; - -#pragma dataflag NOPTR /* dbgvar has no heap pointers */ -static DbgVar dbgvar[] = { - {"allocfreetrace", &runtime·debug.allocfreetrace}, - {"invalidptr", &runtime·invalidptr}, - {"efence", &runtime·debug.efence}, - {"gctrace", &runtime·debug.gctrace}, - {"gcdead", &runtime·debug.gcdead}, - {"scheddetail", &runtime·debug.scheddetail}, - {"schedtrace", &runtime·debug.schedtrace}, - {"scavenge", &runtime·debug.scavenge}, -}; - -void -runtime·parsedebugvars(void) -{ - byte *p; - intgo i, n; - - p = runtime·getenv("GODEBUG"); - if(p != nil){ - for(;;) { - for(i=0; i<nelem(dbgvar); i++) { - n = runtime·findnull((byte*)dbgvar[i].name); - if(runtime·mcmp(p, (byte*)dbgvar[i].name, n) == 0 && p[n] == '=') - *dbgvar[i].value = runtime·atoi(p+n+1); - } - p = runtime·strstr(p, (byte*)","); - if(p == nil) - break; - p++; - } - } - - p = runtime·getenv("GOTRACEBACK"); - if(p == nil) - p = (byte*)""; - if(p[0] == '\0') - traceback_cache = 1<<1; - else if(runtime·strcmp(p, (byte*)"crash") == 0) - traceback_cache = (2<<1) | 1; - else - traceback_cache = runtime·atoi(p)<<1; -} - -// Poor mans 64-bit division. -// This is a very special function, do not use it if you are not sure what you are doing. -// int64 division is lowered into _divv() call on 386, which does not fit into nosplit functions. -// Handles overflow in a time-specific manner. -#pragma textflag NOSPLIT -int32 -runtime·timediv(int64 v, int32 div, int32 *rem) -{ - int32 res, bit; - - res = 0; - for(bit = 30; bit >= 0; bit--) { - if(v >= ((int64)div<<bit)) { - v = v - ((int64)div<<bit); - res += 1<<bit; - } - } - if(v >= (int64)div) { - if(rem != nil) - *rem = 0; - return 0x7fffffff; - } - if(rem != nil) - *rem = v; - return res; -} - -// Helpers for Go. Must be NOSPLIT, must only call NOSPLIT functions, and must not block. - -#pragma textflag NOSPLIT -G* -runtime·getg(void) -{ - return g; -} - -#pragma textflag NOSPLIT -M* -runtime·acquirem(void) -{ - g->m->locks++; - return g->m; -} - -#pragma textflag NOSPLIT -void -runtime·releasem(M *mp) -{ - mp->locks--; - if(mp->locks == 0 && g->preempt) { - // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - } -} - -#pragma textflag NOSPLIT -MCache* -runtime·gomcache(void) -{ - return g->m->mcache; -} - -#pragma textflag NOSPLIT -Slice -reflect·typelinks(void) -{ - extern Type *runtime·typelink[], *runtime·etypelink[]; - Slice ret; - - ret.array = (byte*)runtime·typelink; - ret.len = runtime·etypelink - runtime·typelink; - ret.cap = ret.len; - return ret; -} diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h deleted file mode 100644 index 977c4547d..000000000 --- a/src/runtime/runtime.h +++ /dev/null @@ -1,1130 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* - * basic types - */ -typedef signed char int8; -typedef unsigned char uint8; -typedef signed short int16; -typedef unsigned short uint16; -typedef signed int int32; -typedef unsigned int uint32; -typedef signed long long int int64; -typedef unsigned long long int uint64; -typedef float float32; -typedef double float64; - -#ifdef _64BIT -typedef uint64 uintptr; -typedef int64 intptr; -typedef int64 intgo; // Go's int -typedef uint64 uintgo; // Go's uint -#else -typedef uint32 uintptr; -typedef int32 intptr; -typedef int32 intgo; // Go's int -typedef uint32 uintgo; // Go's uint -#endif - -#ifdef _64BITREG -typedef uint64 uintreg; -#else -typedef uint32 uintreg; -#endif - -/* - * get rid of C types - * the / / / forces a syntax error immediately, - * which will show "last name: XXunsigned". - */ -#define unsigned XXunsigned / / / -#define signed XXsigned / / / -#define char XXchar / / / -#define short XXshort / / / -#define int XXint / / / -#define long XXlong / / / -#define float XXfloat / / / -#define double XXdouble / / / - -/* - * defined types - */ -typedef uint8 bool; -typedef uint8 byte; -typedef struct Func Func; -typedef struct G G; -typedef struct Gobuf Gobuf; -typedef struct SudoG SudoG; -typedef struct Mutex Mutex; -typedef struct M M; -typedef struct P P; -typedef struct SchedT SchedT; -typedef struct Note Note; -typedef struct Slice Slice; -typedef struct String String; -typedef struct FuncVal FuncVal; -typedef struct SigTab SigTab; -typedef struct MCache MCache; -typedef struct FixAlloc FixAlloc; -typedef struct Iface Iface; -typedef struct Itab Itab; -typedef struct InterfaceType InterfaceType; -typedef struct Eface Eface; -typedef struct Type Type; -typedef struct PtrType PtrType; -typedef struct ChanType ChanType; -typedef struct MapType MapType; -typedef struct Defer Defer; -typedef struct Panic Panic; -typedef struct Hmap Hmap; -typedef struct Hiter Hiter; -typedef struct Hchan Hchan; -typedef struct Complex64 Complex64; -typedef struct Complex128 Complex128; -typedef struct LibCall LibCall; -typedef struct WinCallbackContext WinCallbackContext; -typedef struct GCStats GCStats; -typedef struct LFNode LFNode; -typedef struct ParFor ParFor; -typedef struct ParForThread ParForThread; -typedef struct CgoMal CgoMal; -typedef struct PollDesc PollDesc; -typedef struct DebugVars DebugVars; -typedef struct ForceGCState ForceGCState; -typedef struct Stack Stack; - -/* - * Per-CPU declaration. - * - * "extern register" is a special storage class implemented by 6c, 8c, etc. - * On the ARM, it is an actual register; elsewhere it is a slot in thread- - * local storage indexed by a pseudo-register TLS. See zasmhdr in - * src/cmd/dist/buildruntime.c for details, and be aware that the linker may - * make further OS-specific changes to the compiler's output. For example, - * 6l/linux rewrites 0(TLS) as -8(FS). - * - * Every C file linked into a Go program must include runtime.h so that the - * C compiler (6c, 8c, etc.) knows to avoid other uses of these dedicated - * registers. The Go compiler (6g, 8g, etc.) knows to avoid them. - */ -extern register G* g; - -/* - * defined constants - */ -enum -{ - // G status - // - // If you add to this list, add to the list - // of "okay during garbage collection" status - // in mgc0.c too. - Gidle, // 0 - Grunnable, // 1 runnable and on a run queue - Grunning, // 2 - Gsyscall, // 3 - Gwaiting, // 4 - Gmoribund_unused, // 5 currently unused, but hardcoded in gdb scripts - Gdead, // 6 - Genqueue, // 7 Only the Gscanenqueue is used. - Gcopystack, // 8 in this state when newstack is moving the stack - // the following encode that the GC is scanning the stack and what to do when it is done - Gscan = 0x1000, // atomicstatus&~Gscan = the non-scan state, - // Gscanidle = Gscan + Gidle, // Not used. Gidle only used with newly malloced gs - Gscanrunnable = Gscan + Grunnable, // 0x1001 When scanning complets make Grunnable (it is already on run queue) - Gscanrunning = Gscan + Grunning, // 0x1002 Used to tell preemption newstack routine to scan preempted stack. - Gscansyscall = Gscan + Gsyscall, // 0x1003 When scanning completes make is Gsyscall - Gscanwaiting = Gscan + Gwaiting, // 0x1004 When scanning completes make it Gwaiting - // Gscanmoribund_unused, // not possible - // Gscandead, // not possible - Gscanenqueue = Gscan + Genqueue, // When scanning completes make it Grunnable and put on runqueue -}; -enum -{ - // P status - Pidle, - Prunning, - Psyscall, - Pgcstop, - Pdead, -}; -enum -{ - true = 1, - false = 0, -}; -enum -{ - PtrSize = sizeof(void*), -}; -/* - * structures - */ -struct Mutex -{ - // Futex-based impl treats it as uint32 key, - // while sema-based impl as M* waitm. - // Used to be a union, but unions break precise GC. - uintptr key; -}; -struct Note -{ - // Futex-based impl treats it as uint32 key, - // while sema-based impl as M* waitm. - // Used to be a union, but unions break precise GC. - uintptr key; -}; -struct String -{ - byte* str; - intgo len; -}; -struct FuncVal -{ - void (*fn)(void); - // variable-size, fn-specific data here -}; -struct Iface -{ - Itab* tab; - void* data; -}; -struct Eface -{ - Type* type; - void* data; -}; -struct Complex64 -{ - float32 real; - float32 imag; -}; -struct Complex128 -{ - float64 real; - float64 imag; -}; - -struct Slice -{ // must not move anything - byte* array; // actual data - uintgo len; // number of elements - uintgo cap; // allocated number of elements -}; -struct Gobuf -{ - // The offsets of sp, pc, and g are known to (hard-coded in) libmach. - uintptr sp; - uintptr pc; - G* g; - void* ctxt; // this has to be a pointer so that GC scans it - uintreg ret; - uintptr lr; -}; -// Known to compiler. -// Changes here must also be made in src/cmd/gc/select.c's selecttype. -struct SudoG -{ - G* g; - uint32* selectdone; - SudoG* next; - SudoG* prev; - void* elem; // data element - int64 releasetime; - int32 nrelease; // -1 for acquire - SudoG* waitlink; // G.waiting list -}; -struct GCStats -{ - // the struct must consist of only uint64's, - // because it is casted to uint64[]. - uint64 nhandoff; - uint64 nhandoffcnt; - uint64 nprocyield; - uint64 nosyield; - uint64 nsleep; -}; - -struct LibCall -{ - uintptr fn; - uintptr n; // number of parameters - uintptr args; // parameters - uintptr r1; // return values - uintptr r2; - uintptr err; // error number -}; - -// describes how to handle callback -struct WinCallbackContext -{ - void* gobody; // Go function to call - uintptr argsize; // callback arguments size (in bytes) - uintptr restorestack; // adjust stack on return by (in bytes) (386 only) - bool cleanstack; -}; - -// Stack describes a Go execution stack. -// The bounds of the stack are exactly [lo, hi), -// with no implicit data structures on either side. -struct Stack -{ - uintptr lo; - uintptr hi; -}; - -struct G -{ - // Stack parameters. - // stack describes the actual stack memory: [stack.lo, stack.hi). - // stackguard0 is the stack pointer compared in the Go stack growth prologue. - // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. - // stackguard1 is the stack pointer compared in the C stack growth prologue. - // It is stack.lo+StackGuard on g0 and gsignal stacks. - // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). - Stack stack; // offset known to runtime/cgo - uintptr stackguard0; // offset known to liblink - uintptr stackguard1; // offset known to liblink - - Panic* panic; // innermost panic - offset known to liblink - Defer* defer; // innermost defer - Gobuf sched; - uintptr syscallsp; // if status==Gsyscall, syscallsp = sched.sp to use during gc - uintptr syscallpc; // if status==Gsyscall, syscallpc = sched.pc to use during gc - void* param; // passed parameter on wakeup - uint32 atomicstatus; - int64 goid; - int64 waitsince; // approx time when the G become blocked - String waitreason; // if status==Gwaiting - G* schedlink; - bool issystem; // do not output in stack dump, ignore in deadlock detector - bool preempt; // preemption signal, duplicates stackguard0 = StackPreempt - bool paniconfault; // panic (instead of crash) on unexpected fault address - bool preemptscan; // preempted g does scan for GC - bool gcworkdone; // debug: cleared at begining of gc work phase cycle, set by gcphasework, tested at end of cycle - bool throwsplit; // must not split stack - int8 raceignore; // ignore race detection events - M* m; // for debuggers, but offset not hard-coded - M* lockedm; - int32 sig; - Slice writebuf; - uintptr sigcode0; - uintptr sigcode1; - uintptr sigpc; - uintptr gopc; // pc of go statement that created this goroutine - uintptr racectx; - SudoG* waiting; // sudog structures this G is waiting on (that have a valid elem ptr) - uintptr end[]; -}; - -struct M -{ - G* g0; // goroutine with scheduling stack - Gobuf morebuf; // gobuf arg to morestack - - // Fields not known to debuggers. - uint64 procid; // for debuggers, but offset not hard-coded - G* gsignal; // signal-handling G - uintptr tls[4]; // thread-local storage (for x86 extern register) - void (*mstartfn)(void); - G* curg; // current running goroutine - G* caughtsig; // goroutine running during fatal signal - P* p; // attached P for executing Go code (nil if not executing Go code) - P* nextp; - int32 id; - int32 mallocing; - int32 throwing; - int32 gcing; - int32 locks; - int32 softfloat; - int32 dying; - int32 profilehz; - int32 helpgc; - bool spinning; // M is out of work and is actively looking for work - bool blocked; // M is blocked on a Note - uint32 fastrand; - uint64 ncgocall; // number of cgo calls in total - int32 ncgo; // number of cgo calls currently in progress - CgoMal* cgomal; - Note park; - M* alllink; // on allm - M* schedlink; - uint32 machport; // Return address for Mach IPC (OS X) - MCache* mcache; - G* lockedg; - uintptr createstack[32];// Stack that created this thread. - uint32 freglo[16]; // D[i] lsb and F[i] - uint32 freghi[16]; // D[i] msb and F[i+16] - uint32 fflag; // floating point compare flags - uint32 locked; // tracking for LockOSThread - M* nextwaitm; // next M waiting for lock - uintptr waitsema; // semaphore for parking on locks - uint32 waitsemacount; - uint32 waitsemalock; - GCStats gcstats; - bool needextram; - uint8 traceback; - bool (*waitunlockf)(G*, void*); - void* waitlock; - uintptr scalararg[4]; // scalar argument/return for mcall - void* ptrarg[4]; // pointer argument/return for mcall -#ifdef GOOS_windows - uintptr thread; // thread handle - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - LibCall libcall; - uintptr libcallpc; // for cpu profiler - uintptr libcallsp; - G* libcallg; -#endif -#ifdef GOOS_solaris - int32* perrno; // pointer to TLS errno - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - LibCall libcall; - struct MTs { - int64 tv_sec; - int64 tv_nsec; - } ts; - struct MScratch { - uintptr v[6]; - } scratch; -#endif -#ifdef GOOS_plan9 - int8* notesig; - byte* errstr; -#endif - uintptr end[]; -}; - -struct P -{ - Mutex lock; - - int32 id; - uint32 status; // one of Pidle/Prunning/... - P* link; - uint32 schedtick; // incremented on every scheduler call - uint32 syscalltick; // incremented on every system call - M* m; // back-link to associated M (nil if idle) - MCache* mcache; - Defer* deferpool[5]; // pool of available Defer structs of different sizes (see panic.c) - - // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. - uint64 goidcache; - uint64 goidcacheend; - - // Queue of runnable goroutines. - uint32 runqhead; - uint32 runqtail; - G* runq[256]; - - // Available G's (status == Gdead) - G* gfree; - int32 gfreecnt; - - byte pad[64]; -}; - -enum { - // The max value of GOMAXPROCS. - // There are no fundamental restrictions on the value. - MaxGomaxprocs = 1<<8, -}; - -struct SchedT -{ - Mutex lock; - - uint64 goidgen; - - M* midle; // idle m's waiting for work - int32 nmidle; // number of idle m's waiting for work - int32 nmidlelocked; // number of locked m's waiting for work - int32 mcount; // number of m's that have been created - int32 maxmcount; // maximum number of m's allowed (or die) - - P* pidle; // idle P's - uint32 npidle; - uint32 nmspinning; - - // Global runnable queue. - G* runqhead; - G* runqtail; - int32 runqsize; - - // Global cache of dead G's. - Mutex gflock; - G* gfree; - int32 ngfree; - - uint32 gcwaiting; // gc is waiting to run - int32 stopwait; - Note stopnote; - uint32 sysmonwait; - Note sysmonnote; - uint64 lastpoll; - - int32 profilehz; // cpu profiling rate -}; - -// The m->locked word holds two pieces of state counting active calls to LockOSThread/lockOSThread. -// The low bit (LockExternal) is a boolean reporting whether any LockOSThread call is active. -// External locks are not recursive; a second lock is silently ignored. -// The upper bits of m->lockedcount record the nesting depth of calls to lockOSThread -// (counting up by LockInternal), popped by unlockOSThread (counting down by LockInternal). -// Internal locks can be recursive. For instance, a lock for cgo can occur while the main -// goroutine is holding the lock during the initialization phase. -enum -{ - LockExternal = 1, - LockInternal = 2, -}; - -struct SigTab -{ - int32 flags; - int8 *name; -}; -enum -{ - SigNotify = 1<<0, // let signal.Notify have signal, even if from kernel - SigKill = 1<<1, // if signal.Notify doesn't take it, exit quietly - SigThrow = 1<<2, // if signal.Notify doesn't take it, exit loudly - SigPanic = 1<<3, // if the signal is from the kernel, panic - SigDefault = 1<<4, // if the signal isn't explicitly requested, don't monitor it - SigHandling = 1<<5, // our signal handler is registered - SigIgnored = 1<<6, // the signal was ignored before we registered for it - SigGoExit = 1<<7, // cause all runtime procs to exit (only used on Plan 9). -}; - -// Layout of in-memory per-function information prepared by linker -// See http://golang.org/s/go12symtab. -// Keep in sync with linker and with ../../libmach/sym.c -// and with package debug/gosym and with symtab.go in package runtime. -struct Func -{ - uintptr entry; // start pc - int32 nameoff;// function name - - int32 args; // in/out args size - int32 frame; // legacy frame size; use pcsp if possible - - int32 pcsp; - int32 pcfile; - int32 pcln; - int32 npcdata; - int32 nfuncdata; -}; - -// layout of Itab known to compilers -// allocated in non-garbage-collected memory -struct Itab -{ - InterfaceType* inter; - Type* type; - Itab* link; - int32 bad; - int32 unused; - void (*fun[])(void); -}; - -#ifdef GOOS_nacl -enum { - NaCl = 1, -}; -#else -enum { - NaCl = 0, -}; -#endif - -#ifdef GOOS_windows -enum { - Windows = 1 -}; -#else -enum { - Windows = 0 -}; -#endif -#ifdef GOOS_solaris -enum { - Solaris = 1 -}; -#else -enum { - Solaris = 0 -}; -#endif -#ifdef GOOS_plan9 -enum { - Plan9 = 1 -}; -#else -enum { - Plan9 = 0 -}; -#endif - -// Lock-free stack node. -struct LFNode -{ - LFNode *next; - uintptr pushcnt; -}; - -// Parallel for descriptor. -struct ParFor -{ - void (*body)(ParFor*, uint32); // executed for each element - uint32 done; // number of idle threads - uint32 nthr; // total number of threads - uint32 nthrmax; // maximum number of threads - uint32 thrseq; // thread id sequencer - uint32 cnt; // iteration space [0, cnt) - void *ctx; // arbitrary user context - bool wait; // if true, wait while all threads finish processing, - // otherwise parfor may return while other threads are still working - ParForThread *thr; // array of thread descriptors - uint32 pad; // to align ParForThread.pos for 64-bit atomic operations - // stats - uint64 nsteal; - uint64 nstealcnt; - uint64 nprocyield; - uint64 nosyield; - uint64 nsleep; -}; - -// Track memory allocated by code not written in Go during a cgo call, -// so that the garbage collector can see them. -struct CgoMal -{ - CgoMal *next; - void *alloc; -}; - -// Holds variables parsed from GODEBUG env var. -struct DebugVars -{ - int32 allocfreetrace; - int32 efence; - int32 gctrace; - int32 gcdead; - int32 scheddetail; - int32 schedtrace; - int32 scavenge; -}; - -// Indicates to write barrier and sychronization task to preform. -enum -{ // Synchronization Write barrier - GCoff, // stop and start nop - GCquiesce, // stop and start nop - GCstw, // stop the ps nop - GCmark, // scan the stacks and start no white to black - GCsweep, // stop and start nop -}; - -struct ForceGCState -{ - Mutex lock; - G* g; - uint32 idle; -}; - -extern uint32 runtime·gcphase; - -/* - * defined macros - * you need super-gopher-guru privilege - * to add this list. - */ -#define nelem(x) (sizeof(x)/sizeof((x)[0])) -#define nil ((void*)0) -#define offsetof(s,m) (uint32)(&(((s*)0)->m)) -#define ROUND(x, n) (((x)+(n)-1)&~(uintptr)((n)-1)) /* all-caps to mark as macro: it evaluates n twice */ - -/* - * known to compiler - */ -enum { - Structrnd = sizeof(uintreg), -}; - -byte* runtime·startup_random_data; -uint32 runtime·startup_random_data_len; - -int32 runtime·invalidptr; - -enum { - // hashinit wants this many random bytes - HashRandomBytes = 32 -}; - -uint32 runtime·readgstatus(G*); -void runtime·casgstatus(G*, uint32, uint32); -void runtime·quiesce(G*); -bool runtime·stopg(G*); -void runtime·restartg(G*); -void runtime·gcphasework(G*); - -/* - * deferred subroutine calls - */ -struct Defer -{ - int32 siz; - bool started; - uintptr argp; // where args were copied from - uintptr pc; - FuncVal* fn; - Panic* panic; // panic that is running defer - Defer* link; -}; - -// argp used in Defer structs when there is no argp. -#define NoArgs ((uintptr)-1) - -/* - * panics - */ -struct Panic -{ - void* argp; // pointer to arguments of deferred call run during panic; cannot move - known to liblink - Eface arg; // argument to panic - Panic* link; // link to earlier panic - bool recovered; // whether this panic is over - bool aborted; // the panic was aborted -}; - -/* - * stack traces - */ -typedef struct Stkframe Stkframe; -typedef struct BitVector BitVector; -struct Stkframe -{ - Func* fn; // function being run - uintptr pc; // program counter within fn - uintptr continpc; // program counter where execution can continue, or 0 if not - uintptr lr; // program counter at caller aka link register - uintptr sp; // stack pointer at pc - uintptr fp; // stack pointer at caller aka frame pointer - uintptr varp; // top of local variables - uintptr argp; // pointer to function arguments - uintptr arglen; // number of bytes at argp - BitVector* argmap; // force use of this argmap -}; - -enum -{ - TraceRuntimeFrames = 1<<0, // include frames for internal runtime functions. - TraceTrap = 1<<1, // the initial PC, SP are from a trap, not a return PC from a call -}; -intgo runtime·gentraceback(uintptr, uintptr, uintptr, G*, intgo, uintptr*, intgo, bool(**)(Stkframe*, void*), void*, uintgo); -void runtime·tracebackdefers(G*, bool(**)(Stkframe*, void*), void*); -void runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G* gp); -void runtime·tracebacktrap(uintptr pc, uintptr sp, uintptr lr, G* gp); -void runtime·tracebackothers(G*); -bool runtime·haszeroargs(uintptr pc); -bool runtime·topofstack(Func*); -enum -{ - // The maximum number of frames we print for a traceback - TracebackMaxFrames = 100, -}; - -/* - * external data - */ -extern String runtime·emptystring; -extern G** runtime·allg; -extern Slice runtime·allgs; // []*G -extern uintptr runtime·allglen; -extern G* runtime·lastg; -extern M* runtime·allm; -extern P* runtime·allp[MaxGomaxprocs+1]; -extern int32 runtime·gomaxprocs; -extern uint32 runtime·needextram; -extern uint32 runtime·panicking; -extern int8* runtime·goos; -extern int32 runtime·ncpu; -extern bool runtime·iscgo; -extern void (*runtime·sysargs)(int32, uint8**); -extern uintptr runtime·maxstring; -extern uint32 runtime·cpuid_ecx; -extern uint32 runtime·cpuid_edx; -extern DebugVars runtime·debug; -extern uintptr runtime·maxstacksize; -extern Note runtime·signote; -extern ForceGCState runtime·forcegc; -extern SchedT runtime·sched; -extern int32 runtime·newprocs; - -/* - * common functions and data - */ -int32 runtime·strcmp(byte*, byte*); -int32 runtime·strncmp(byte*, byte*, uintptr); -byte* runtime·strstr(byte*, byte*); -intgo runtime·findnull(byte*); -intgo runtime·findnullw(uint16*); -void runtime·dump(byte*, int32); -int32 runtime·runetochar(byte*, int32); -int32 runtime·charntorune(int32*, uint8*, int32); - - -/* - * This macro is used when writing C functions - * called as if they were Go functions. - * Passed the address of a result before a return statement, - * it makes sure the result has been flushed to memory - * before the return. - * - * It is difficult to write such functions portably, because - * of the varying requirements on the alignment of the - * first output value. Almost all code should write such - * functions in .goc files, where goc2c (part of cmd/dist) - * can arrange the correct alignment for the target system. - * Goc2c also takes care of conveying to the garbage collector - * which parts of the argument list are inputs vs outputs. - * - * Therefore, do NOT use this macro if at all possible. - */ -#define FLUSH(x) USED(x) - -/* - * GoOutput is a type with the same alignment requirements as the - * initial output argument from a Go function. Only for use in cases - * where using goc2c is not possible. See comment on FLUSH above. - */ -typedef uint64 GoOutput; - -void runtime·gogo(Gobuf*); -void runtime·gostartcall(Gobuf*, void(*)(void), void*); -void runtime·gostartcallfn(Gobuf*, FuncVal*); -void runtime·gosave(Gobuf*); -void runtime·goargs(void); -void runtime·goenvs(void); -void runtime·goenvs_unix(void); -void* runtime·getu(void); -void runtime·throw(int8*); -bool runtime·canpanic(G*); -void runtime·prints(int8*); -void runtime·printf(int8*, ...); -void runtime·snprintf(byte*, int32, int8*, ...); -byte* runtime·mchr(byte*, byte, byte*); -int32 runtime·mcmp(byte*, byte*, uintptr); -void runtime·memmove(void*, void*, uintptr); -String runtime·catstring(String, String); -String runtime·gostring(byte*); -Slice runtime·makeStringSlice(intgo); -String runtime·gostringn(byte*, intgo); -Slice runtime·gobytes(byte*, intgo); -String runtime·gostringnocopy(byte*); -String runtime·gostringw(uint16*); -void runtime·initsig(void); -void runtime·sigenable(uint32 sig); -void runtime·sigdisable(uint32 sig); -int32 runtime·gotraceback(bool *crash); -void runtime·goroutineheader(G*); -int32 runtime·open(int8*, int32, int32); -int32 runtime·read(int32, void*, int32); -int32 runtime·write(uintptr, void*, int32); // use uintptr to accommodate windows. -int32 runtime·close(int32); -int32 runtime·mincore(void*, uintptr, byte*); -void runtime·jmpdefer(FuncVal*, uintptr); -void runtime·exit1(int32); -void runtime·ready(G*); -byte* runtime·getenv(int8*); -int32 runtime·atoi(byte*); -void runtime·newosproc(M *mp, void *stk); -void runtime·mstart(void); -G* runtime·malg(int32); -void runtime·asminit(void); -void runtime·mpreinit(M*); -void runtime·minit(void); -void runtime·unminit(void); -void runtime·signalstack(byte*, int32); -void runtime·tracebackinit(void); -void runtime·symtabinit(void); -Func* runtime·findfunc(uintptr); -int32 runtime·funcline(Func*, uintptr, String*); -int32 runtime·funcspdelta(Func*, uintptr); -int8* runtime·funcname(Func*); -int32 runtime·pcdatavalue(Func*, int32, uintptr); -void runtime·stackinit(void); -Stack runtime·stackalloc(uint32); -void runtime·stackfree(Stack); -void runtime·shrinkstack(G*); -void runtime·shrinkfinish(void); -MCache* runtime·allocmcache(void); -void runtime·freemcache(MCache*); -void runtime·mallocinit(void); -void runtime·gcinit(void); -void* runtime·mallocgc(uintptr size, Type* typ, uint32 flag); -void runtime·runpanic(Panic*); -uintptr runtime·getcallersp(void*); -int32 runtime·mcount(void); -int32 runtime·gcount(void); -void runtime·mcall(void(**)(G*)); -void runtime·onM(void(**)(void)); -void runtime·onMsignal(void(**)(void)); -uint32 runtime·fastrand1(void); -void runtime·rewindmorestack(Gobuf*); -int32 runtime·timediv(int64, int32, int32*); -int32 runtime·round2(int32 x); // round x up to a power of 2. - -// atomic operations -bool runtime·cas(uint32*, uint32, uint32); -bool runtime·cas64(uint64*, uint64, uint64); -bool runtime·casp(void**, void*, void*); -// Don't confuse with XADD x86 instruction, -// this one is actually 'addx', that is, add-and-fetch. -uint32 runtime·xadd(uint32 volatile*, int32); -uint64 runtime·xadd64(uint64 volatile*, int64); -uint32 runtime·xchg(uint32 volatile*, uint32); -uint64 runtime·xchg64(uint64 volatile*, uint64); -void* runtime·xchgp(void* volatile*, void*); -uint32 runtime·atomicload(uint32 volatile*); -void runtime·atomicstore(uint32 volatile*, uint32); -void runtime·atomicstore64(uint64 volatile*, uint64); -uint64 runtime·atomicload64(uint64 volatile*); -void* runtime·atomicloadp(void* volatile*); -uintptr runtime·atomicloaduintptr(uintptr volatile*); -void runtime·atomicstorep(void* volatile*, void*); -void runtime·atomicstoreuintptr(uintptr volatile*, uintptr); -void runtime·atomicor8(byte volatile*, byte); - -void runtime·setg(G*); -void runtime·newextram(void); -void runtime·exit(int32); -void runtime·breakpoint(void); -void runtime·gosched_m(G*); -void runtime·schedtrace(bool); -void runtime·park(bool(*)(G*, void*), void*, String); -void runtime·parkunlock(Mutex*, String); -void runtime·tsleep(int64, String); -M* runtime·newm(void); -void runtime·goexit(void); -void runtime·asmcgocall(void (*fn)(void*), void*); -int32 runtime·asmcgocall_errno(void (*fn)(void*), void*); -void runtime·entersyscall(void); -void runtime·reentersyscall(uintptr, uintptr); -void runtime·entersyscallblock(void); -void runtime·exitsyscall(void); -G* runtime·newproc1(FuncVal*, byte*, int32, int32, void*); -bool runtime·sigsend(int32 sig); -intgo runtime·callers(intgo, uintptr*, intgo); -intgo runtime·gcallers(G*, intgo, uintptr*, intgo); -int64 runtime·nanotime(void); // monotonic time -int64 runtime·unixnanotime(void); // real time, can skip -void runtime·dopanic(int32); -void runtime·startpanic(void); -void runtime·freezetheworld(void); -void runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp); -void runtime·resetcpuprofiler(int32); -void runtime·setcpuprofilerate(int32); -void runtime·usleep(uint32); -int64 runtime·cputicks(void); -int64 runtime·tickspersecond(void); -void runtime·blockevent(int64, intgo); -G* runtime·netpoll(bool); -void runtime·netpollready(G**, PollDesc*, int32); -uintptr runtime·netpollfd(PollDesc*); -void** runtime·netpolluser(PollDesc*); -bool runtime·netpollclosing(PollDesc*); -void runtime·netpolllock(PollDesc*); -void runtime·netpollunlock(PollDesc*); -void runtime·crash(void); -void runtime·parsedebugvars(void); -void* runtime·funcdata(Func*, int32); -void runtime·setmaxthreads_m(void); -G* runtime·timejump(void); -void runtime·iterate_itabs(void (**callback)(Itab*)); -void runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*)); - -#pragma varargck argpos runtime·printf 1 -#pragma varargck type "c" int32 -#pragma varargck type "d" int32 -#pragma varargck type "d" uint32 -#pragma varargck type "D" int64 -#pragma varargck type "D" uint64 -#pragma varargck type "x" int32 -#pragma varargck type "x" uint32 -#pragma varargck type "X" int64 -#pragma varargck type "X" uint64 -#pragma varargck type "p" void* -#pragma varargck type "p" uintptr -#pragma varargck type "s" int8* -#pragma varargck type "s" uint8* -#pragma varargck type "S" String - -void runtime·stoptheworld(void); -void runtime·starttheworld(void); -extern uint32 runtime·worldsema; - -/* - * mutual exclusion locks. in the uncontended case, - * as fast as spin locks (just a few user-level instructions), - * but on the contention path they sleep in the kernel. - * a zeroed Mutex is unlocked (no need to initialize each lock). - */ -void runtime·lock(Mutex*); -void runtime·unlock(Mutex*); - -/* - * sleep and wakeup on one-time events. - * before any calls to notesleep or notewakeup, - * must call noteclear to initialize the Note. - * then, exactly one thread can call notesleep - * and exactly one thread can call notewakeup (once). - * once notewakeup has been called, the notesleep - * will return. future notesleep will return immediately. - * subsequent noteclear must be called only after - * previous notesleep has returned, e.g. it's disallowed - * to call noteclear straight after notewakeup. - * - * notetsleep is like notesleep but wakes up after - * a given number of nanoseconds even if the event - * has not yet happened. if a goroutine uses notetsleep to - * wake up early, it must wait to call noteclear until it - * can be sure that no other goroutine is calling - * notewakeup. - * - * notesleep/notetsleep are generally called on g0, - * notetsleepg is similar to notetsleep but is called on user g. - */ -void runtime·noteclear(Note*); -void runtime·notesleep(Note*); -void runtime·notewakeup(Note*); -bool runtime·notetsleep(Note*, int64); // false - timeout -bool runtime·notetsleepg(Note*, int64); // false - timeout - -/* - * low-level synchronization for implementing the above - */ -uintptr runtime·semacreate(void); -int32 runtime·semasleep(int64); -void runtime·semawakeup(M*); -// or -void runtime·futexsleep(uint32*, uint32, int64); -void runtime·futexwakeup(uint32*, uint32); - -/* - * Mutex-free stack. - * Initialize uint64 head to 0, compare with 0 to test for emptiness. - * The stack does not keep pointers to nodes, - * so they can be garbage collected if there are no other pointers to nodes. - */ -void runtime·lfstackpush(uint64 *head, LFNode *node); -LFNode* runtime·lfstackpop(uint64 *head); - -/* - * Parallel for over [0, n). - * body() is executed for each iteration. - * nthr - total number of worker threads. - * ctx - arbitrary user context. - * if wait=true, threads return from parfor() when all work is done; - * otherwise, threads can return while other threads are still finishing processing. - */ -ParFor* runtime·parforalloc(uint32 nthrmax); -void runtime·parforsetup(ParFor *desc, uint32 nthr, uint32 n, void *ctx, bool wait, void (*body)(ParFor*, uint32)); -void runtime·parfordo(ParFor *desc); -void runtime·parforiters(ParFor*, uintptr, uintptr*, uintptr*); - -/* - * low level C-called - */ -// for mmap, we only pass the lower 32 bits of file offset to the -// assembly routine; the higher bits (if required), should be provided -// by the assembly routine as 0. -uint8* runtime·mmap(byte*, uintptr, int32, int32, int32, uint32); -void runtime·munmap(byte*, uintptr); -void runtime·madvise(byte*, uintptr, int32); -void runtime·memclr(byte*, uintptr); -void runtime·setcallerpc(void*, void*); -void* runtime·getcallerpc(void*); -void runtime·printbool(bool); -void runtime·printbyte(int8); -void runtime·printfloat(float64); -void runtime·printint(int64); -void runtime·printiface(Iface); -void runtime·printeface(Eface); -void runtime·printstring(String); -void runtime·printpc(void*); -void runtime·printpointer(void*); -void runtime·printuint(uint64); -void runtime·printhex(uint64); -void runtime·printslice(Slice); -void runtime·printcomplex(Complex128); - -/* - * runtime go-called - */ -void runtime·gopanic(Eface); -void runtime·panicindex(void); -void runtime·panicslice(void); -void runtime·panicdivide(void); - -/* - * runtime c-called (but written in Go) - */ -void runtime·printany(Eface); -void runtime·newTypeAssertionError(String*, String*, String*, String*, Eface*); -void runtime·fadd64c(uint64, uint64, uint64*); -void runtime·fsub64c(uint64, uint64, uint64*); -void runtime·fmul64c(uint64, uint64, uint64*); -void runtime·fdiv64c(uint64, uint64, uint64*); -void runtime·fneg64c(uint64, uint64*); -void runtime·f32to64c(uint32, uint64*); -void runtime·f64to32c(uint64, uint32*); -void runtime·fcmp64c(uint64, uint64, int32*, bool*); -void runtime·fintto64c(int64, uint64*); -void runtime·f64tointc(uint64, int64*, bool*); - -/* - * wrapped for go users - */ -float64 runtime·Inf(int32 sign); -float64 runtime·NaN(void); -float32 runtime·float32frombits(uint32 i); -uint32 runtime·float32tobits(float32 f); -float64 runtime·float64frombits(uint64 i); -uint64 runtime·float64tobits(float64 f); -float64 runtime·frexp(float64 d, int32 *ep); -bool runtime·isInf(float64 f, int32 sign); -bool runtime·isNaN(float64 f); -float64 runtime·ldexp(float64 d, int32 e); -float64 runtime·modf(float64 d, float64 *ip); -void runtime·semacquire(uint32*, bool); -void runtime·semrelease(uint32*); -int32 runtime·gomaxprocsfunc(int32 n); -void runtime·procyield(uint32); -void runtime·osyield(void); -void runtime·lockOSThread(void); -void runtime·unlockOSThread(void); - -bool runtime·showframe(Func*, G*); -void runtime·printcreatedby(G*); - -void runtime·ifaceE2I(InterfaceType*, Eface, Iface*); -bool runtime·ifaceE2I2(InterfaceType*, Eface, Iface*); -uintptr runtime·memlimit(void); - -// float.c -extern float64 runtime·nan; -extern float64 runtime·posinf; -extern float64 runtime·neginf; -extern uint64 ·nan; -extern uint64 ·posinf; -extern uint64 ·neginf; -#define ISNAN(f) ((f) != (f)) - -enum -{ - UseSpanType = 1, -}; diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go new file mode 100644 index 000000000..15dea01a3 --- /dev/null +++ b/src/runtime/runtime1.go @@ -0,0 +1,417 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Keep a cached value to make gotraceback fast, +// since we call it on every call to gentraceback. +// The cached value is a uint32 in which the low bit +// is the "crash" setting and the top 31 bits are the +// gotraceback value. +var traceback_cache uint32 = 2 << 1 + +// The GOTRACEBACK environment variable controls the +// behavior of a Go program that is crashing and exiting. +// GOTRACEBACK=0 suppress all tracebacks +// GOTRACEBACK=1 default behavior - show tracebacks but exclude runtime frames +// GOTRACEBACK=2 show tracebacks including runtime frames +// GOTRACEBACK=crash show tracebacks including runtime frames, then crash (core dump etc) +//go:nosplit +func gotraceback(crash *bool) int32 { + _g_ := getg() + if crash != nil { + *crash = false + } + if _g_.m.traceback != 0 { + return int32(_g_.m.traceback) + } + if crash != nil { + *crash = traceback_cache&1 != 0 + } + return int32(traceback_cache >> 1) +} + +var ( + argc int32 + argv **byte +) + +// nosplit for use in linux/386 startup linux_setup_vdso +//go:nosplit +func argv_index(argv **byte, i int32) *byte { + return *(**byte)(add(unsafe.Pointer(argv), uintptr(i)*ptrSize)) +} + +func args(c int32, v **byte) { + argc = c + argv = v + sysargs(c, v) +} + +var ( + // TODO: Retire in favor of GOOS== checks. + isplan9 int32 + issolaris int32 + iswindows int32 +) + +// Information about what cpu features are available. +// Set on startup in asm_{x86/amd64}.s. +var ( +//cpuid_ecx uint32 +//cpuid_edx uint32 +) + +func goargs() { + if GOOS == "windows" { + return + } + + argslice = make([]string, argc) + for i := int32(0); i < argc; i++ { + argslice[i] = gostringnocopy(argv_index(argv, i)) + } +} + +func goenvs_unix() { + n := int32(0) + for argv_index(argv, argc+1+n) != nil { + n++ + } + + envs = make([]string, n) + for i := int32(0); i < n; i++ { + envs[i] = gostringnocopy(argv_index(argv, argc+1+i)) + } +} + +func environ() []string { + return envs +} + +func testAtomic64() { + var z64, x64 uint64 + + z64 = 42 + x64 = 0 + // TODO: PREFETCH((unsafe.Pointer)(&z64)) + if cas64(&z64, x64, 1) { + gothrow("cas64 failed") + } + if x64 != 0 { + gothrow("cas64 failed") + } + x64 = 42 + if !cas64(&z64, x64, 1) { + gothrow("cas64 failed") + } + if x64 != 42 || z64 != 1 { + gothrow("cas64 failed") + } + if atomicload64(&z64) != 1 { + gothrow("load64 failed") + } + atomicstore64(&z64, (1<<40)+1) + if atomicload64(&z64) != (1<<40)+1 { + gothrow("store64 failed") + } + if xadd64(&z64, (1<<40)+1) != (2<<40)+2 { + gothrow("xadd64 failed") + } + if atomicload64(&z64) != (2<<40)+2 { + gothrow("xadd64 failed") + } + if xchg64(&z64, (3<<40)+3) != (2<<40)+2 { + gothrow("xchg64 failed") + } + if atomicload64(&z64) != (3<<40)+3 { + gothrow("xchg64 failed") + } +} + +func check() { + var ( + a int8 + b uint8 + c int16 + d uint16 + e int32 + f uint32 + g int64 + h uint64 + i, i1 float32 + j, j1 float64 + k, k1 unsafe.Pointer + l *uint16 + m [4]byte + ) + type x1t struct { + x uint8 + } + type y1t struct { + x1 x1t + y uint8 + } + var x1 x1t + var y1 y1t + + if unsafe.Sizeof(a) != 1 { + gothrow("bad a") + } + if unsafe.Sizeof(b) != 1 { + gothrow("bad b") + } + if unsafe.Sizeof(c) != 2 { + gothrow("bad c") + } + if unsafe.Sizeof(d) != 2 { + gothrow("bad d") + } + if unsafe.Sizeof(e) != 4 { + gothrow("bad e") + } + if unsafe.Sizeof(f) != 4 { + gothrow("bad f") + } + if unsafe.Sizeof(g) != 8 { + gothrow("bad g") + } + if unsafe.Sizeof(h) != 8 { + gothrow("bad h") + } + if unsafe.Sizeof(i) != 4 { + gothrow("bad i") + } + if unsafe.Sizeof(j) != 8 { + gothrow("bad j") + } + if unsafe.Sizeof(k) != ptrSize { + gothrow("bad k") + } + if unsafe.Sizeof(l) != ptrSize { + gothrow("bad l") + } + if unsafe.Sizeof(x1) != 1 { + gothrow("bad unsafe.Sizeof x1") + } + if unsafe.Offsetof(y1.y) != 1 { + gothrow("bad offsetof y1.y") + } + if unsafe.Sizeof(y1) != 2 { + gothrow("bad unsafe.Sizeof y1") + } + + if timediv(12345*1000000000+54321, 1000000000, &e) != 12345 || e != 54321 { + gothrow("bad timediv") + } + + var z uint32 + z = 1 + if !cas(&z, 1, 2) { + gothrow("cas1") + } + if z != 2 { + gothrow("cas2") + } + + z = 4 + if cas(&z, 5, 6) { + gothrow("cas3") + } + if z != 4 { + gothrow("cas4") + } + + z = 0xffffffff + if !cas(&z, 0xffffffff, 0xfffffffe) { + gothrow("cas5") + } + if z != 0xfffffffe { + gothrow("cas6") + } + + k = unsafe.Pointer(uintptr(0xfedcb123)) + if ptrSize == 8 { + k = unsafe.Pointer(uintptr(unsafe.Pointer(k)) << 10) + } + if casp(&k, nil, nil) { + gothrow("casp1") + } + k1 = add(k, 1) + if !casp(&k, k, k1) { + gothrow("casp2") + } + if k != k1 { + gothrow("casp3") + } + + m = [4]byte{1, 1, 1, 1} + atomicor8(&m[1], 0xf0) + if m[0] != 1 || m[1] != 0xf1 || m[2] != 1 || m[3] != 1 { + gothrow("atomicor8") + } + + *(*uint64)(unsafe.Pointer(&j)) = ^uint64(0) + if j == j { + gothrow("float64nan") + } + if !(j != j) { + gothrow("float64nan1") + } + + *(*uint64)(unsafe.Pointer(&j1)) = ^uint64(1) + if j == j1 { + gothrow("float64nan2") + } + if !(j != j1) { + gothrow("float64nan3") + } + + *(*uint32)(unsafe.Pointer(&i)) = ^uint32(0) + if i == i { + gothrow("float32nan") + } + if i == i { + gothrow("float32nan1") + } + + *(*uint32)(unsafe.Pointer(&i1)) = ^uint32(1) + if i == i1 { + gothrow("float32nan2") + } + if i == i1 { + gothrow("float32nan3") + } + + testAtomic64() + + if _FixedStack != round2(_FixedStack) { + gothrow("FixedStack is not power-of-2") + } +} + +type dbgVar struct { + name string + value *int32 +} + +// Do we report invalid pointers found during stack or heap scans? +//var invalidptr int32 = 1 + +var dbgvars = []dbgVar{ + {"allocfreetrace", &debug.allocfreetrace}, + {"invalidptr", &invalidptr}, + {"efence", &debug.efence}, + {"gctrace", &debug.gctrace}, + {"gcdead", &debug.gcdead}, + {"scheddetail", &debug.scheddetail}, + {"schedtrace", &debug.schedtrace}, + {"scavenge", &debug.scavenge}, +} + +func parsedebugvars() { + for p := gogetenv("GODEBUG"); p != ""; { + field := "" + i := index(p, ",") + if i < 0 { + field, p = p, "" + } else { + field, p = p[:i], p[i+1:] + } + i = index(field, "=") + if i < 0 { + continue + } + key, value := field[:i], field[i+1:] + for _, v := range dbgvars { + if v.name == key { + *v.value = int32(goatoi(value)) + } + } + } + + switch p := gogetenv("GOTRACEBACK"); p { + case "": + traceback_cache = 1 << 1 + case "crash": + traceback_cache = 2<<1 | 1 + default: + traceback_cache = uint32(goatoi(p)) << 1 + } +} + +// Poor mans 64-bit division. +// This is a very special function, do not use it if you are not sure what you are doing. +// int64 division is lowered into _divv() call on 386, which does not fit into nosplit functions. +// Handles overflow in a time-specific manner. +//go:nosplit +func timediv(v int64, div int32, rem *int32) int32 { + res := int32(0) + for bit := 30; bit >= 0; bit-- { + if v >= int64(div)<<uint(bit) { + v = v - (int64(div) << uint(bit)) + res += 1 << uint(bit) + } + } + if v >= int64(div) { + if rem != nil { + *rem = 0 + } + return 0x7fffffff + } + if rem != nil { + *rem = int32(v) + } + return res +} + +// Helpers for Go. Must be NOSPLIT, must only call NOSPLIT functions, and must not block. + +//go:nosplit +func acquirem() *m { + _g_ := getg() + _g_.m.locks++ + return _g_.m +} + +//go:nosplit +func releasem(mp *m) { + _g_ := getg() + mp.locks-- + if mp.locks == 0 && _g_.preempt { + // restore the preemption request in case we've cleared it in newstack + _g_.stackguard0 = stackPreempt + } +} + +//go:nosplit +func gomcache() *mcache { + return getg().m.mcache +} + +var typelink, etypelink [0]byte + +//go:nosplit +func typelinks() []*_type { + var ret []*_type + sp := (*slice)(unsafe.Pointer(&ret)) + sp.array = (*byte)(unsafe.Pointer(&typelink)) + sp.len = uint((uintptr(unsafe.Pointer(&etypelink)) - uintptr(unsafe.Pointer(&typelink))) / unsafe.Sizeof(ret[0])) + sp.cap = sp.len + return ret +} + +// TODO: move back into mgc0.c when converted to Go +func readgogc() int32 { + p := gogetenv("GOGC") + if p == "" { + return 100 + } + if p == "off" { + return -1 + } + return int32(goatoi(p)) +} diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go new file mode 100644 index 000000000..7987a7373 --- /dev/null +++ b/src/runtime/runtime2.go @@ -0,0 +1,606 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +/* + * defined constants + */ +const ( + // G status + // + // If you add to this list, add to the list + // of "okay during garbage collection" status + // in mgc0.c too. + _Gidle = iota // 0 + _Grunnable // 1 runnable and on a run queue + _Grunning // 2 + _Gsyscall // 3 + _Gwaiting // 4 + _Gmoribund_unused // 5 currently unused, but hardcoded in gdb scripts + _Gdead // 6 + _Genqueue // 7 Only the Gscanenqueue is used. + _Gcopystack // 8 in this state when newstack is moving the stack + // the following encode that the GC is scanning the stack and what to do when it is done + _Gscan = 0x1000 // atomicstatus&~Gscan = the non-scan state, + // _Gscanidle = _Gscan + _Gidle, // Not used. Gidle only used with newly malloced gs + _Gscanrunnable = _Gscan + _Grunnable // 0x1001 When scanning complets make Grunnable (it is already on run queue) + _Gscanrunning = _Gscan + _Grunning // 0x1002 Used to tell preemption newstack routine to scan preempted stack. + _Gscansyscall = _Gscan + _Gsyscall // 0x1003 When scanning completes make is Gsyscall + _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 When scanning completes make it Gwaiting + // _Gscanmoribund_unused, // not possible + // _Gscandead, // not possible + _Gscanenqueue = _Gscan + _Genqueue // When scanning completes make it Grunnable and put on runqueue +) + +const ( + // P status + _Pidle = iota + _Prunning + _Psyscall + _Pgcstop + _Pdead +) + +// The next line makes 'go generate' write the zgen_*.go files with +// per-OS and per-arch information, including constants +// named goos_$GOOS and goarch_$GOARCH for every +// known GOOS and GOARCH. The constant is 1 on the +// current system, 0 otherwise; multiplying by them is +// useful for defining GOOS- or GOARCH-specific constants. +//go:generate go run gengoos.go + +type mutex struct { + // Futex-based impl treats it as uint32 key, + // while sema-based impl as M* waitm. + // Used to be a union, but unions break precise GC. + key uintptr +} + +type note struct { + // Futex-based impl treats it as uint32 key, + // while sema-based impl as M* waitm. + // Used to be a union, but unions break precise GC. + key uintptr +} + +type _string struct { + str *byte + len int +} + +type funcval struct { + fn uintptr + // variable-size, fn-specific data here +} + +type iface struct { + tab *itab + data unsafe.Pointer +} + +type eface struct { + _type *_type + data unsafe.Pointer +} + +type slice struct { + array *byte // actual data + len uint // number of elements + cap uint // allocated number of elements +} + +type gobuf struct { + // The offsets of sp, pc, and g are known to (hard-coded in) libmach. + sp uintptr + pc uintptr + g *g + ctxt unsafe.Pointer // this has to be a pointer so that gc scans it + ret uintreg + lr uintptr +} + +// Known to compiler. +// Changes here must also be made in src/cmd/gc/select.c's selecttype. +type sudog struct { + g *g + selectdone *uint32 + next *sudog + prev *sudog + elem unsafe.Pointer // data element + releasetime int64 + nrelease int32 // -1 for acquire + waitlink *sudog // g.waiting list +} + +type gcstats struct { + // the struct must consist of only uint64's, + // because it is casted to uint64[]. + nhandoff uint64 + nhandoffcnt uint64 + nprocyield uint64 + nosyield uint64 + nsleep uint64 +} + +type libcall struct { + fn uintptr + n uintptr // number of parameters + args uintptr // parameters + r1 uintptr // return values + r2 uintptr + err uintptr // error number +} + +// describes how to handle callback +type wincallbackcontext struct { + gobody unsafe.Pointer // go function to call + argsize uintptr // callback arguments size (in bytes) + restorestack uintptr // adjust stack on return by (in bytes) (386 only) + cleanstack bool +} + +// Stack describes a Go execution stack. +// The bounds of the stack are exactly [lo, hi), +// with no implicit data structures on either side. +type stack struct { + lo uintptr + hi uintptr +} + +type g struct { + // Stack parameters. + // stack describes the actual stack memory: [stack.lo, stack.hi). + // stackguard0 is the stack pointer compared in the Go stack growth prologue. + // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. + // stackguard1 is the stack pointer compared in the C stack growth prologue. + // It is stack.lo+StackGuard on g0 and gsignal stacks. + // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). + stack stack // offset known to runtime/cgo + stackguard0 uintptr // offset known to liblink + stackguard1 uintptr // offset known to liblink + + _panic *_panic // innermost panic - offset known to liblink + _defer *_defer // innermost defer + sched gobuf + syscallsp uintptr // if status==gsyscall, syscallsp = sched.sp to use during gc + syscallpc uintptr // if status==gsyscall, syscallpc = sched.pc to use during gc + param unsafe.Pointer // passed parameter on wakeup + atomicstatus uint32 + goid int64 + waitsince int64 // approx time when the g become blocked + waitreason string // if status==gwaiting + schedlink *g + issystem bool // do not output in stack dump, ignore in deadlock detector + preempt bool // preemption signal, duplicates stackguard0 = stackpreempt + paniconfault bool // panic (instead of crash) on unexpected fault address + preemptscan bool // preempted g does scan for gc + gcworkdone bool // debug: cleared at begining of gc work phase cycle, set by gcphasework, tested at end of cycle + throwsplit bool // must not split stack + raceignore int8 // ignore race detection events + m *m // for debuggers, but offset not hard-coded + lockedm *m + sig uint32 + writebuf []byte + sigcode0 uintptr + sigcode1 uintptr + sigpc uintptr + gopc uintptr // pc of go statement that created this goroutine + racectx uintptr + waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr) + end [0]byte +} + +type mts struct { + tv_sec int64 + tv_nsec int64 +} + +type mscratch struct { + v [6]uintptr +} + +type m struct { + g0 *g // goroutine with scheduling stack + morebuf gobuf // gobuf arg to morestack + + // Fields not known to debuggers. + procid uint64 // for debuggers, but offset not hard-coded + gsignal *g // signal-handling g + tls [4]uintptr // thread-local storage (for x86 extern register) + mstartfn unsafe.Pointer // todo go func() + curg *g // current running goroutine + caughtsig *g // goroutine running during fatal signal + p *p // attached p for executing go code (nil if not executing go code) + nextp *p + id int32 + mallocing int32 + throwing int32 + gcing int32 + locks int32 + softfloat int32 + dying int32 + profilehz int32 + helpgc int32 + spinning bool // m is out of work and is actively looking for work + blocked bool // m is blocked on a note + fastrand uint32 + ncgocall uint64 // number of cgo calls in total + ncgo int32 // number of cgo calls currently in progress + cgomal *cgomal + park note + alllink *m // on allm + schedlink *m + machport uint32 // return address for mach ipc (os x) + mcache *mcache + lockedg *g + createstack [32]uintptr // stack that created this thread. + freglo [16]uint32 // d[i] lsb and f[i] + freghi [16]uint32 // d[i] msb and f[i+16] + fflag uint32 // floating point compare flags + locked uint32 // tracking for lockosthread + nextwaitm *m // next m waiting for lock + waitsema uintptr // semaphore for parking on locks + waitsemacount uint32 + waitsemalock uint32 + gcstats gcstats + needextram bool + traceback uint8 + waitunlockf unsafe.Pointer // todo go func(*g, unsafe.pointer) bool + waitlock unsafe.Pointer + //#ifdef GOOS_windows + thread uintptr // thread handle + // these are here because they are too large to be on the stack + // of low-level NOSPLIT functions. + libcall libcall + libcallpc uintptr // for cpu profiler + libcallsp uintptr + libcallg *g + //#endif + //#ifdef GOOS_solaris + perrno *int32 // pointer to tls errno + // these are here because they are too large to be on the stack + // of low-level NOSPLIT functions. + //LibCall libcall; + ts mts + scratch mscratch + //#endif + //#ifdef GOOS_plan9 + notesig *int8 + errstr *byte + //#endif + end [0]byte +} + +type p struct { + lock mutex + + id int32 + status uint32 // one of pidle/prunning/... + link *p + schedtick uint32 // incremented on every scheduler call + syscalltick uint32 // incremented on every system call + m *m // back-link to associated m (nil if idle) + mcache *mcache + deferpool [5]*_defer // pool of available defer structs of different sizes (see panic.c) + + // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. + goidcache uint64 + goidcacheend uint64 + + // Queue of runnable goroutines. + runqhead uint32 + runqtail uint32 + runq [256]*g + + // Available G's (status == Gdead) + gfree *g + gfreecnt int32 + + pad [64]byte +} + +const ( + // The max value of GOMAXPROCS. + // There are no fundamental restrictions on the value. + _MaxGomaxprocs = 1 << 8 +) + +type schedt struct { + lock mutex + + goidgen uint64 + + midle *m // idle m's waiting for work + nmidle int32 // number of idle m's waiting for work + nmidlelocked int32 // number of locked m's waiting for work + mcount int32 // number of m's that have been created + maxmcount int32 // maximum number of m's allowed (or die) + + pidle *p // idle p's + npidle uint32 + nmspinning uint32 + + // Global runnable queue. + runqhead *g + runqtail *g + runqsize int32 + + // Global cache of dead G's. + gflock mutex + gfree *g + ngfree int32 + + gcwaiting uint32 // gc is waiting to run + stopwait int32 + stopnote note + sysmonwait uint32 + sysmonnote note + lastpoll uint64 + + profilehz int32 // cpu profiling rate +} + +// The m->locked word holds two pieces of state counting active calls to LockOSThread/lockOSThread. +// The low bit (LockExternal) is a boolean reporting whether any LockOSThread call is active. +// External locks are not recursive; a second lock is silently ignored. +// The upper bits of m->lockedcount record the nesting depth of calls to lockOSThread +// (counting up by LockInternal), popped by unlockOSThread (counting down by LockInternal). +// Internal locks can be recursive. For instance, a lock for cgo can occur while the main +// goroutine is holding the lock during the initialization phase. +const ( + _LockExternal = 1 + _LockInternal = 2 +) + +type sigtabtt struct { + flags int32 + name *int8 +} + +const ( + _SigNotify = 1 << 0 // let signal.Notify have signal, even if from kernel + _SigKill = 1 << 1 // if signal.Notify doesn't take it, exit quietly + _SigThrow = 1 << 2 // if signal.Notify doesn't take it, exit loudly + _SigPanic = 1 << 3 // if the signal is from the kernel, panic + _SigDefault = 1 << 4 // if the signal isn't explicitly requested, don't monitor it + _SigHandling = 1 << 5 // our signal handler is registered + _SigIgnored = 1 << 6 // the signal was ignored before we registered for it + _SigGoExit = 1 << 7 // cause all runtime procs to exit (only used on Plan 9). +) + +// Layout of in-memory per-function information prepared by linker +// See http://golang.org/s/go12symtab. +// Keep in sync with linker and with ../../libmach/sym.c +// and with package debug/gosym and with symtab.go in package runtime. +type _func struct { + entry uintptr // start pc + nameoff int32 // function name + + args int32 // in/out args size + frame int32 // legacy frame size; use pcsp if possible + + pcsp int32 + pcfile int32 + pcln int32 + npcdata int32 + nfuncdata int32 +} + +// layout of Itab known to compilers +// allocated in non-garbage-collected memory +type itab struct { + inter *interfacetype + _type *_type + link *itab + bad int32 + unused int32 + fun [0]uintptr +} + +// Lock-free stack node. +type lfnode struct { + next *lfnode + pushcnt uintptr +} + +// Parallel for descriptor. +type parfor struct { + body unsafe.Pointer // go func(*parfor, uint32), executed for each element + done uint32 // number of idle threads + nthr uint32 // total number of threads + nthrmax uint32 // maximum number of threads + thrseq uint32 // thread id sequencer + cnt uint32 // iteration space [0, cnt) + ctx unsafe.Pointer // arbitrary user context + wait bool // if true, wait while all threads finish processing, + // otherwise parfor may return while other threads are still working + thr *parforthread // array of thread descriptors + pad uint32 // to align parforthread.pos for 64-bit atomic operations + // stats + nsteal uint64 + nstealcnt uint64 + nprocyield uint64 + nosyield uint64 + nsleep uint64 +} + +// Track memory allocated by code not written in Go during a cgo call, +// so that the garbage collector can see them. +type cgomal struct { + next *cgomal + alloc unsafe.Pointer +} + +// Holds variables parsed from GODEBUG env var. +type debugvars struct { + allocfreetrace int32 + efence int32 + gctrace int32 + gcdead int32 + scheddetail int32 + schedtrace int32 + scavenge int32 +} + +// Indicates to write barrier and sychronization task to preform. +const ( + _GCoff = iota // stop and start nop + _GCquiesce // stop and start nop + _GCstw // stop the ps nop + _GCmark // scan the stacks and start no white to black + _GCsweep // stop and start nop +) + +type forcegcstate struct { + lock mutex + g *g + idle uint32 +} + +var gcphase uint32 + +/* + * known to compiler + */ +const ( + _Structrnd = regSize +) + +var startup_random_data *byte +var startup_random_data_len uint32 + +var invalidptr int32 + +const ( + // hashinit wants this many random bytes + _HashRandomBytes = 32 +) + +/* + * deferred subroutine calls + */ +type _defer struct { + siz int32 + started bool + argp uintptr // where args were copied from + pc uintptr + fn *funcval + _panic *_panic // panic that is running defer + link *_defer +} + +/* + * panics + */ +type _panic struct { + argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink + arg interface{} // argument to panic + link *_panic // link to earlier panic + recovered bool // whether this panic is over + aborted bool // the panic was aborted +} + +/* + * stack traces + */ + +type stkframe struct { + fn *_func // function being run + pc uintptr // program counter within fn + continpc uintptr // program counter where execution can continue, or 0 if not + lr uintptr // program counter at caller aka link register + sp uintptr // stack pointer at pc + fp uintptr // stack pointer at caller aka frame pointer + varp uintptr // top of local variables + argp uintptr // pointer to function arguments + arglen uintptr // number of bytes at argp + argmap *bitvector // force use of this argmap +} + +const ( + _TraceRuntimeFrames = 1 << 0 // include frames for internal runtime functions. + _TraceTrap = 1 << 1 // the initial PC, SP are from a trap, not a return PC from a call +) + +const ( + // The maximum number of frames we print for a traceback + _TracebackMaxFrames = 100 +) + +var ( + emptystring string + allg **g + allglen uintptr + lastg *g + allm *m + allp [_MaxGomaxprocs + 1]*p + gomaxprocs int32 + needextram uint32 + panicking uint32 + goos *int8 + ncpu int32 + iscgo bool + cpuid_ecx uint32 + cpuid_edx uint32 + debug debugvars + signote note + forcegc forcegcstate + sched schedt + newprocs int32 +) + +/* + * mutual exclusion locks. in the uncontended case, + * as fast as spin locks (just a few user-level instructions), + * but on the contention path they sleep in the kernel. + * a zeroed Mutex is unlocked (no need to initialize each lock). + */ + +/* + * sleep and wakeup on one-time events. + * before any calls to notesleep or notewakeup, + * must call noteclear to initialize the Note. + * then, exactly one thread can call notesleep + * and exactly one thread can call notewakeup (once). + * once notewakeup has been called, the notesleep + * will return. future notesleep will return immediately. + * subsequent noteclear must be called only after + * previous notesleep has returned, e.g. it's disallowed + * to call noteclear straight after notewakeup. + * + * notetsleep is like notesleep but wakes up after + * a given number of nanoseconds even if the event + * has not yet happened. if a goroutine uses notetsleep to + * wake up early, it must wait to call noteclear until it + * can be sure that no other goroutine is calling + * notewakeup. + * + * notesleep/notetsleep are generally called on g0, + * notetsleepg is similar to notetsleep but is called on user g. + */ +// bool runtime·notetsleep(Note*, int64); // false - timeout +// bool runtime·notetsleepg(Note*, int64); // false - timeout + +/* + * Lock-free stack. + * Initialize uint64 head to 0, compare with 0 to test for emptiness. + * The stack does not keep pointers to nodes, + * so they can be garbage collected if there are no other pointers to nodes. + */ + +/* + * Parallel for over [0, n). + * body() is executed for each iteration. + * nthr - total number of worker threads. + * ctx - arbitrary user context. + * if wait=true, threads return from parfor() when all work is done; + * otherwise, threads can return while other threads are still finishing processing. + */ + +// for mmap, we only pass the lower 32 bits of file offset to the +// assembly routine; the higher bits (if required), should be provided +// by the assembly routine as 0. diff --git a/src/runtime/runtime2_windows.go b/src/runtime/runtime2_windows.go new file mode 100644 index 000000000..80fc386e9 --- /dev/null +++ b/src/runtime/runtime2_windows.go @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// TODO(brainman): move generation of zsys_windows_*.s out from cmd/dist/buildruntime.c and into here +const cb_max = 2000 // maximum number of windows callbacks allowed (must be in sync with MAXWINCB from cmd/dist/buildruntime.c) diff --git a/src/runtime/select.go b/src/runtime/select.go index f735a71e2..6717b9395 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -167,8 +167,8 @@ func selunlock(sel *_select) { } } -func selparkcommit(gp *g, sel *_select) bool { - selunlock(sel) +func selparkcommit(gp *g, sel unsafe.Pointer) bool { + selunlock((*_select)(sel)) return true } @@ -363,7 +363,7 @@ loop: // wait for someone to wake us up gp.param = nil - gopark(unsafe.Pointer(funcPC(selparkcommit)), unsafe.Pointer(sel), "select") + gopark(selparkcommit, unsafe.Pointer(sel), "select") // someone woke us up sellock(sel) diff --git a/src/runtime/signal.c b/src/runtime/signal.c deleted file mode 100644 index 0674bfb22..000000000 --- a/src/runtime/signal.c +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" - -void -runtime·sigenable_m(void) -{ - uint32 s; - - s = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - runtime·sigenable(s); -} - -void -runtime·sigdisable_m(void) -{ - uint32 s; - - s = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - runtime·sigdisable(s); -} diff --git a/src/runtime/signal1_unix.go b/src/runtime/signal1_unix.go new file mode 100644 index 000000000..25f01e056 --- /dev/null +++ b/src/runtime/signal1_unix.go @@ -0,0 +1,111 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package runtime + +const ( + _SIG_DFL uintptr = 0 + _SIG_IGN uintptr = 1 +) + +func initsig() { + // _NSIG is the number of signals on this operating system. + // sigtable should describe what to do for all the possible signals. + if len(sigtable) != _NSIG { + print("runtime: len(sigtable)=", len(sigtable), " _NSIG=", _NSIG, "\n") + gothrow("initsig") + } + + // First call: basic setup. + for i := int32(0); i < _NSIG; i++ { + t := &sigtable[i] + if t.flags == 0 || t.flags&_SigDefault != 0 { + continue + } + + // For some signals, we respect an inherited SIG_IGN handler + // rather than insist on installing our own default handler. + // Even these signals can be fetched using the os/signal package. + switch i { + case _SIGHUP, _SIGINT: + if getsig(i) == _SIG_IGN { + t.flags = _SigNotify | _SigIgnored + continue + } + } + + t.flags |= _SigHandling + setsig(i, funcPC(sighandler), true) + } +} + +func sigenable(sig uint32) { + if sig >= uint32(len(sigtable)) { + return + } + + t := &sigtable[sig] + if t.flags&_SigNotify != 0 && t.flags&_SigHandling == 0 { + t.flags |= _SigHandling + if getsig(int32(sig)) == _SIG_IGN { + t.flags |= _SigIgnored + } + setsig(int32(sig), funcPC(sighandler), true) + } +} + +func sigdisable(sig uint32) { + if sig >= uint32(len(sigtable)) { + return + } + + t := &sigtable[sig] + if t.flags&_SigNotify != 0 && t.flags&_SigHandling != 0 { + t.flags &^= _SigHandling + if t.flags&_SigIgnored != 0 { + setsig(int32(sig), _SIG_IGN, true) + } else { + setsig(int32(sig), _SIG_DFL, true) + } + } +} + +func resetcpuprofiler(hz int32) { + var it itimerval + if hz == 0 { + setitimer(_ITIMER_PROF, &it, nil) + } else { + it.it_interval.tv_sec = 0 + it.it_interval.set_usec(1000000 / hz) + it.it_value = it.it_interval + setitimer(_ITIMER_PROF, &it, nil) + } + _g_ := getg() + _g_.m.profilehz = hz +} + +func sigpipe() { + setsig(_SIGPIPE, _SIG_DFL, false) + raise(_SIGPIPE) +} + +func crash() { + if GOOS == "darwin" { + // OS X core dumps are linear dumps of the mapped memory, + // from the first virtual byte to the last, with zeros in the gaps. + // Because of the way we arrange the address space on 64-bit systems, + // this means the OS X core file will be >128 GB and even on a zippy + // workstation can take OS X well over an hour to write (uninterruptible). + // Save users from making that mistake. + if ptrSize == 8 { + return + } + } + + unblocksignals() + setsig(_SIGABRT, _SIG_DFL, false) + raise(_SIGABRT) +} diff --git a/src/runtime/signal_386.c b/src/runtime/signal_386.c deleted file mode 100644 index 30a7488bd..000000000 --- a/src/runtime/signal_386.c +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_GOOS_GOARCH.h" -#include "signals_GOOS.h" - -void -runtime·dumpregs(Siginfo *info, void *ctxt) -{ - USED(info); - USED(ctxt); - - runtime·printf("eax %x\n", SIG_EAX(info, ctxt)); - runtime·printf("ebx %x\n", SIG_EBX(info, ctxt)); - runtime·printf("ecx %x\n", SIG_ECX(info, ctxt)); - runtime·printf("edx %x\n", SIG_EDX(info, ctxt)); - runtime·printf("edi %x\n", SIG_EDI(info, ctxt)); - runtime·printf("esi %x\n", SIG_ESI(info, ctxt)); - runtime·printf("ebp %x\n", SIG_EBP(info, ctxt)); - runtime·printf("esp %x\n", SIG_ESP(info, ctxt)); - runtime·printf("eip %x\n", SIG_EIP(info, ctxt)); - runtime·printf("eflags %x\n", SIG_EFLAGS(info, ctxt)); - runtime·printf("cs %x\n", SIG_CS(info, ctxt)); - runtime·printf("fs %x\n", SIG_FS(info, ctxt)); - runtime·printf("gs %x\n", SIG_GS(info, ctxt)); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) -{ - uintptr *sp; - SigTab *t; - bool crash; - - if(sig == SIGPROF) { - runtime·sigprof((byte*)SIG_EIP(info, ctxt), (byte*)SIG_ESP(info, ctxt), nil, gp, g->m); - return; - } - - t = &runtime·sigtab[sig]; - if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = SIG_CODE0(info, ctxt); - gp->sigcode1 = SIG_CODE1(info, ctxt); - gp->sigpc = SIG_EIP(info, ctxt); - -#ifdef GOOS_darwin - // Work around Leopard bug that doesn't set FPE_INTDIV. - // Look at instruction to see if it is a divide. - // Not necessary in Snow Leopard (si_code will be != 0). - if(sig == SIGFPE && gp->sigcode0 == 0) { - byte *pc; - pc = (byte*)gp->sigpc; - if(pc[0] == 0x66) // 16-bit instruction prefix - pc++; - if(pc[0] == 0xF6 || pc[0] == 0xF7) - gp->sigcode0 = FPE_INTDIV; - } -#endif - - // Only push runtime·sigpanic if eip != 0. - // If eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(SIG_EIP(info, ctxt) != 0) { - sp = (uintptr*)SIG_ESP(info, ctxt); - *--sp = SIG_EIP(info, ctxt); - SIG_ESP(info, ctxt) = (uintptr)sp; - } - SIG_EIP(info, ctxt) = (uintptr)runtime·sigpanic; - return; - } - - if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - - g->m->throwing = 1; - g->m->caughtsig = gp; - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%x\n", SIG_EIP(info, ctxt)); - if(g->m->lockedg != nil && g->m->ncgo > 0 && gp == g->m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = g->m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback(&crash)){ - runtime·goroutineheader(gp); - runtime·tracebacktrap(SIG_EIP(info, ctxt), SIG_ESP(info, ctxt), 0, gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(info, ctxt); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); -} diff --git a/src/runtime/signal_386.go b/src/runtime/signal_386.go new file mode 100644 index 000000000..5336a4330 --- /dev/null +++ b/src/runtime/signal_386.go @@ -0,0 +1,131 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd + +package runtime + +import "unsafe" + +func dumpregs(c *sigctxt) { + print("eax ", hex(c.eax()), "\n") + print("ebx ", hex(c.ebx()), "\n") + print("ecx ", hex(c.ecx()), "\n") + print("edx ", hex(c.edx()), "\n") + print("edi ", hex(c.edi()), "\n") + print("esi ", hex(c.esi()), "\n") + print("ebp ", hex(c.ebp()), "\n") + print("esp ", hex(c.esp()), "\n") + print("eip ", hex(c.eip()), "\n") + print("eflags ", hex(c.eflags()), "\n") + print("cs ", hex(c.cs()), "\n") + print("fs ", hex(c.fs()), "\n") + print("gs ", hex(c.gs()), "\n") +} + +func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { + _g_ := getg() + c := &sigctxt{info, ctxt} + + if sig == _SIGPROF { + sigprof((*byte)(unsafe.Pointer(uintptr(c.eip()))), (*byte)(unsafe.Pointer(uintptr(c.esp()))), nil, gp, _g_.m) + return + } + + flags := int32(_SigThrow) + if sig < uint32(len(sigtable)) { + flags = sigtable[sig].flags + } + if c.sigcode() != _SI_USER && flags&_SigPanic != 0 { + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = sig + gp.sigcode0 = uintptr(c.sigcode()) + gp.sigcode1 = uintptr(c.sigaddr()) + gp.sigpc = uintptr(c.eip()) + + if GOOS == "darwin" { + // Work around Leopard bug that doesn't set FPE_INTDIV. + // Look at instruction to see if it is a divide. + // Not necessary in Snow Leopard (si_code will be != 0). + if sig == _SIGFPE && gp.sigcode0 == 0 { + pc := (*[4]byte)(unsafe.Pointer(gp.sigpc)) + i := 0 + if pc[i] == 0x66 { // 16-bit instruction prefix + i++ + } + if pc[i] == 0xF6 || pc[i] == 0xF7 { + gp.sigcode0 = _FPE_INTDIV + } + } + } + + // Only push runtime.sigpanic if rip != 0. + // If rip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime.sigpanic instead. + // (Otherwise the trace will end at runtime.sigpanic and we + // won't get to see who faulted.) + if c.eip() != 0 { + sp := c.esp() + if regSize > ptrSize { + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(uintptr(sp))) = 0 + } + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(uintptr(sp))) = uintptr(c.eip()) + c.set_esp(sp) + } + c.set_eip(uint32(funcPC(sigpanic))) + return + } + + if c.sigcode() == _SI_USER || flags&_SigNotify != 0 { + if sigsend(sig) { + return + } + } + + if flags&_SigKill != 0 { + exit(2) + } + + if flags&_SigThrow == 0 { + return + } + + _g_.m.throwing = 1 + _g_.m.caughtsig = gp + startpanic() + + if sig < uint32(len(sigtable)) { + print(sigtable[sig].name, "\n") + } else { + print("Signal ", sig, "\n") + } + + print("PC=", hex(c.eip()), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + goroutineheader(gp) + tracebacktrap(uintptr(c.eip()), uintptr(c.esp()), 0, gp) + tracebackothers(gp) + print("\n") + dumpregs(c) + } + + if docrash { + crash() + } + + exit(2) +} diff --git a/src/runtime/signal_amd64x.c b/src/runtime/signal_amd64x.c deleted file mode 100644 index feb4afcce..000000000 --- a/src/runtime/signal_amd64x.c +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build amd64 amd64p32 -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_GOOS_GOARCH.h" -#include "signals_GOOS.h" - -void -runtime·dumpregs(Siginfo *info, void *ctxt) -{ - USED(info); - USED(ctxt); - - runtime·printf("rax %X\n", SIG_RAX(info, ctxt)); - runtime·printf("rbx %X\n", SIG_RBX(info, ctxt)); - runtime·printf("rcx %X\n", SIG_RCX(info, ctxt)); - runtime·printf("rdx %X\n", SIG_RDX(info, ctxt)); - runtime·printf("rdi %X\n", SIG_RDI(info, ctxt)); - runtime·printf("rsi %X\n", SIG_RSI(info, ctxt)); - runtime·printf("rbp %X\n", SIG_RBP(info, ctxt)); - runtime·printf("rsp %X\n", SIG_RSP(info, ctxt)); - runtime·printf("r8 %X\n", SIG_R8(info, ctxt) ); - runtime·printf("r9 %X\n", SIG_R9(info, ctxt) ); - runtime·printf("r10 %X\n", SIG_R10(info, ctxt)); - runtime·printf("r11 %X\n", SIG_R11(info, ctxt)); - runtime·printf("r12 %X\n", SIG_R12(info, ctxt)); - runtime·printf("r13 %X\n", SIG_R13(info, ctxt)); - runtime·printf("r14 %X\n", SIG_R14(info, ctxt)); - runtime·printf("r15 %X\n", SIG_R15(info, ctxt)); - runtime·printf("rip %X\n", SIG_RIP(info, ctxt)); - runtime·printf("rflags %X\n", SIG_RFLAGS(info, ctxt)); - runtime·printf("cs %X\n", SIG_CS(info, ctxt)); - runtime·printf("fs %X\n", SIG_FS(info, ctxt)); - runtime·printf("gs %X\n", SIG_GS(info, ctxt)); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) -{ - uintptr *sp; - SigTab *t; - bool crash; - - if(sig == SIGPROF) { - runtime·sigprof((byte*)SIG_RIP(info, ctxt), (byte*)SIG_RSP(info, ctxt), nil, gp, g->m); - return; - } - -#ifdef GOOS_darwin - // x86-64 has 48-bit virtual addresses. The top 16 bits must echo bit 47. - // The hardware delivers a different kind of fault for a malformed address - // than it does for an attempt to access a valid but unmapped address. - // OS X 10.9.2 mishandles the malformed address case, making it look like - // a user-generated signal (like someone ran kill -SEGV ourpid). - // We pass user-generated signals to os/signal, or else ignore them. - // Doing that here - and returning to the faulting code - results in an - // infinite loop. It appears the best we can do is rewrite what the kernel - // delivers into something more like the truth. The address used below - // has very little chance of being the one that caused the fault, but it is - // malformed, it is clearly not a real pointer, and if it does get printed - // in real life, people will probably search for it and find this code. - // There are no Google hits for b01dfacedebac1e or 0xb01dfacedebac1e - // as I type this comment. - if(sig == SIGSEGV && SIG_CODE0(info, ctxt) == SI_USER) { - SIG_CODE0(info, ctxt) = SI_USER+1; - info->si_addr = (void*)(uintptr)0xb01dfacedebac1eULL; - } -#endif - - t = &runtime·sigtab[sig]; - if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = SIG_CODE0(info, ctxt); - gp->sigcode1 = SIG_CODE1(info, ctxt); - gp->sigpc = SIG_RIP(info, ctxt); - -#ifdef GOOS_darwin - // Work around Leopard bug that doesn't set FPE_INTDIV. - // Look at instruction to see if it is a divide. - // Not necessary in Snow Leopard (si_code will be != 0). - if(sig == SIGFPE && gp->sigcode0 == 0) { - byte *pc; - pc = (byte*)gp->sigpc; - if((pc[0]&0xF0) == 0x40) // 64-bit REX prefix - pc++; - else if(pc[0] == 0x66) // 16-bit instruction prefix - pc++; - if(pc[0] == 0xF6 || pc[0] == 0xF7) - gp->sigcode0 = FPE_INTDIV; - } -#endif - - // Only push runtime·sigpanic if rip != 0. - // If rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(SIG_RIP(info, ctxt) != 0) { - sp = (uintptr*)SIG_RSP(info, ctxt); - if(sizeof(uintreg) > sizeof(uintptr)) - *--sp = 0; - *--sp = SIG_RIP(info, ctxt); - SIG_RSP(info, ctxt) = (uintptr)sp; - } - SIG_RIP(info, ctxt) = (uintptr)runtime·sigpanic; - return; - } - - if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - - g->m->throwing = 1; - g->m->caughtsig = gp; - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", SIG_RIP(info, ctxt)); - if(g->m->lockedg != nil && g->m->ncgo > 0 && gp == g->m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = g->m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback(&crash)){ - runtime·goroutineheader(gp); - runtime·tracebacktrap(SIG_RIP(info, ctxt), SIG_RSP(info, ctxt), 0, gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(info, ctxt); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); -} diff --git a/src/runtime/signal_amd64x.go b/src/runtime/signal_amd64x.go new file mode 100644 index 000000000..de88d93a5 --- /dev/null +++ b/src/runtime/signal_amd64x.go @@ -0,0 +1,163 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 amd64p32 +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +package runtime + +import "unsafe" + +func dumpregs(c *sigctxt) { + print("rax ", hex(c.rax()), "\n") + print("rbx ", hex(c.rbx()), "\n") + print("rcx ", hex(c.rcx()), "\n") + print("rdx ", hex(c.rdx()), "\n") + print("rdi ", hex(c.rdi()), "\n") + print("rsi ", hex(c.rsi()), "\n") + print("rbp ", hex(c.rbp()), "\n") + print("rsp ", hex(c.rsp()), "\n") + print("r8 ", hex(c.r8()), "\n") + print("r9 ", hex(c.r9()), "\n") + print("r10 ", hex(c.r10()), "\n") + print("r11 ", hex(c.r11()), "\n") + print("r12 ", hex(c.r12()), "\n") + print("r13 ", hex(c.r13()), "\n") + print("r14 ", hex(c.r14()), "\n") + print("r15 ", hex(c.r15()), "\n") + print("rip ", hex(c.rip()), "\n") + print("rflags ", hex(c.rflags()), "\n") + print("cs ", hex(c.cs()), "\n") + print("fs ", hex(c.fs()), "\n") + print("gs ", hex(c.gs()), "\n") +} + +func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { + _g_ := getg() + c := &sigctxt{info, ctxt} + + if sig == _SIGPROF { + sigprof((*byte)(unsafe.Pointer(uintptr(c.rip()))), (*byte)(unsafe.Pointer(uintptr(c.rsp()))), nil, gp, _g_.m) + return + } + + if GOOS == "darwin" { + // x86-64 has 48-bit virtual addresses. The top 16 bits must echo bit 47. + // The hardware delivers a different kind of fault for a malformed address + // than it does for an attempt to access a valid but unmapped address. + // OS X 10.9.2 mishandles the malformed address case, making it look like + // a user-generated signal (like someone ran kill -SEGV ourpid). + // We pass user-generated signals to os/signal, or else ignore them. + // Doing that here - and returning to the faulting code - results in an + // infinite loop. It appears the best we can do is rewrite what the kernel + // delivers into something more like the truth. The address used below + // has very little chance of being the one that caused the fault, but it is + // malformed, it is clearly not a real pointer, and if it does get printed + // in real life, people will probably search for it and find this code. + // There are no Google hits for b01dfacedebac1e or 0xb01dfacedebac1e + // as I type this comment. + if sig == _SIGSEGV && c.sigcode() == _SI_USER { + c.set_sigcode(_SI_USER + 1) + c.set_sigaddr(0xb01dfacedebac1e) + } + } + + flags := int32(_SigThrow) + if sig < uint32(len(sigtable)) { + flags = sigtable[sig].flags + } + if c.sigcode() != _SI_USER && flags&_SigPanic != 0 { + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = sig + gp.sigcode0 = uintptr(c.sigcode()) + gp.sigcode1 = uintptr(c.sigaddr()) + gp.sigpc = uintptr(c.rip()) + + if GOOS == "darwin" { + // Work around Leopard bug that doesn't set FPE_INTDIV. + // Look at instruction to see if it is a divide. + // Not necessary in Snow Leopard (si_code will be != 0). + if sig == _SIGFPE && gp.sigcode0 == 0 { + pc := (*[4]byte)(unsafe.Pointer(gp.sigpc)) + i := 0 + if pc[i]&0xF0 == 0x40 { // 64-bit REX prefix + i++ + } else if pc[i] == 0x66 { // 16-bit instruction prefix + i++ + } + if pc[i] == 0xF6 || pc[i] == 0xF7 { + gp.sigcode0 = _FPE_INTDIV + } + } + } + + // Only push runtime.sigpanic if rip != 0. + // If rip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime.sigpanic instead. + // (Otherwise the trace will end at runtime.sigpanic and we + // won't get to see who faulted.) + if c.rip() != 0 { + sp := c.rsp() + if regSize > ptrSize { + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(uintptr(sp))) = 0 + } + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(uintptr(sp))) = uintptr(c.rip()) + c.set_rsp(sp) + } + c.set_rip(uint64(funcPC(sigpanic))) + return + } + + if c.sigcode() == _SI_USER || flags&_SigNotify != 0 { + if sigsend(sig) { + return + } + } + + if flags&_SigKill != 0 { + exit(2) + } + + if flags&_SigThrow == 0 { + return + } + + _g_.m.throwing = 1 + _g_.m.caughtsig = gp + startpanic() + + if sig < uint32(len(sigtable)) { + print(sigtable[sig].name, "\n") + } else { + print("Signal ", sig, "\n") + } + + print("PC=", hex(c.rip()), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + goroutineheader(gp) + tracebacktrap(uintptr(c.rip()), uintptr(c.rsp()), 0, gp) + tracebackothers(gp) + print("\n") + dumpregs(c) + } + + if docrash { + crash() + } + + exit(2) +} diff --git a/src/runtime/signal_arm.c b/src/runtime/signal_arm.c deleted file mode 100644 index afad5e7d1..000000000 --- a/src/runtime/signal_arm.c +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_GOOS_GOARCH.h" -#include "signals_GOOS.h" - -void -runtime·dumpregs(Siginfo *info, void *ctxt) -{ - USED(info); - USED(ctxt); - - runtime·printf("trap %x\n", SIG_TRAP(info, ctxt)); - runtime·printf("error %x\n", SIG_ERROR(info, ctxt)); - runtime·printf("oldmask %x\n", SIG_OLDMASK(info, ctxt)); - runtime·printf("r0 %x\n", SIG_R0(info, ctxt)); - runtime·printf("r1 %x\n", SIG_R1(info, ctxt)); - runtime·printf("r2 %x\n", SIG_R2(info, ctxt)); - runtime·printf("r3 %x\n", SIG_R3(info, ctxt)); - runtime·printf("r4 %x\n", SIG_R4(info, ctxt)); - runtime·printf("r5 %x\n", SIG_R5(info, ctxt)); - runtime·printf("r6 %x\n", SIG_R6(info, ctxt)); - runtime·printf("r7 %x\n", SIG_R7(info, ctxt)); - runtime·printf("r8 %x\n", SIG_R8(info, ctxt)); - runtime·printf("r9 %x\n", SIG_R9(info, ctxt)); - runtime·printf("r10 %x\n", SIG_R10(info, ctxt)); - runtime·printf("fp %x\n", SIG_FP(info, ctxt)); - runtime·printf("ip %x\n", SIG_IP(info, ctxt)); - runtime·printf("sp %x\n", SIG_SP(info, ctxt)); - runtime·printf("lr %x\n", SIG_LR(info, ctxt)); - runtime·printf("pc %x\n", SIG_PC(info, ctxt)); - runtime·printf("cpsr %x\n", SIG_CPSR(info, ctxt)); - runtime·printf("fault %x\n", SIG_FAULT(info, ctxt)); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) -{ - SigTab *t; - bool crash; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)SIG_PC(info, ctxt), (uint8*)SIG_SP(info, ctxt), (uint8*)SIG_LR(info, ctxt), gp, g->m); - return; - } - - t = &runtime·sigtab[sig]; - if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = SIG_CODE0(info, ctxt); - gp->sigcode1 = SIG_FAULT(info, ctxt); - gp->sigpc = SIG_PC(info, ctxt); - - // We arrange lr, and pc to pretend the panicking - // function calls sigpanic directly. - // Always save LR to stack so that panics in leaf - // functions are correctly handled. This smashes - // the stack frame but we're not going back there - // anyway. - SIG_SP(info, ctxt) -= 4; - *(uint32*)SIG_SP(info, ctxt) = SIG_LR(info, ctxt); - // Don't bother saving PC if it's zero, which is - // probably a call to a nil func: the old link register - // is more useful in the stack trace. - if(gp->sigpc != 0) - SIG_LR(info, ctxt) = gp->sigpc; - // In case we are panicking from external C code - SIG_R10(info, ctxt) = (uintptr)gp; - SIG_PC(info, ctxt) = (uintptr)runtime·sigpanic; - return; - } - - if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - - g->m->throwing = 1; - g->m->caughtsig = gp; - if(runtime·panicking) // traceback already printed - runtime·exit(2); - runtime·panicking = 1; - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%x\n", SIG_PC(info, ctxt)); - if(g->m->lockedg != nil && g->m->ncgo > 0 && gp == g->m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = g->m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback(&crash)){ - runtime·goroutineheader(gp); - runtime·tracebacktrap(SIG_PC(info, ctxt), SIG_SP(info, ctxt), SIG_LR(info, ctxt), gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(info, ctxt); - } - - if(crash) - runtime·crash(); - - runtime·exit(2); -} diff --git a/src/runtime/signal_arm.go b/src/runtime/signal_arm.go new file mode 100644 index 000000000..d224ce645 --- /dev/null +++ b/src/runtime/signal_arm.go @@ -0,0 +1,126 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd + +package runtime + +import "unsafe" + +func dumpregs(c *sigctxt) { + print("trap ", hex(c.trap()), "\n") + print("error ", hex(c.error()), "\n") + print("oldmask ", hex(c.oldmask()), "\n") + print("r0 ", hex(c.r0()), "\n") + print("r1 ", hex(c.r1()), "\n") + print("r2 ", hex(c.r2()), "\n") + print("r3 ", hex(c.r3()), "\n") + print("r4 ", hex(c.r4()), "\n") + print("r5 ", hex(c.r5()), "\n") + print("r6 ", hex(c.r6()), "\n") + print("r7 ", hex(c.r7()), "\n") + print("r8 ", hex(c.r8()), "\n") + print("r9 ", hex(c.r9()), "\n") + print("r10 ", hex(c.r10()), "\n") + print("fp ", hex(c.fp()), "\n") + print("ip ", hex(c.ip()), "\n") + print("sp ", hex(c.sp()), "\n") + print("lr ", hex(c.lr()), "\n") + print("pc ", hex(c.pc()), "\n") + print("cpsr ", hex(c.cpsr()), "\n") + print("fault ", hex(c.fault()), "\n") +} + +func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { + _g_ := getg() + c := &sigctxt{info, ctxt} + + if sig == _SIGPROF { + sigprof((*byte)(unsafe.Pointer(uintptr(c.pc()))), (*byte)(unsafe.Pointer(uintptr(c.sp()))), (*byte)(unsafe.Pointer(uintptr(c.lr()))), gp, _g_.m) + return + } + + flags := int32(_SigThrow) + if sig < uint32(len(sigtable)) { + flags = sigtable[sig].flags + } + if c.sigcode() != _SI_USER && flags&_SigPanic != 0 { + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = sig + gp.sigcode0 = uintptr(c.sigcode()) + gp.sigcode1 = uintptr(c.fault()) + gp.sigpc = uintptr(c.pc()) + + // We arrange lr, and pc to pretend the panicking + // function calls sigpanic directly. + // Always save LR to stack so that panics in leaf + // functions are correctly handled. This smashes + // the stack frame but we're not going back there + // anyway. + sp := c.sp() - 4 + c.set_sp(sp) + *(*uint32)(unsafe.Pointer(uintptr(sp))) = c.lr() + + // Don't bother saving PC if it's zero, which is + // probably a call to a nil func: the old link register + // is more useful in the stack trace. + if gp.sigpc != 0 { + c.set_lr(uint32(gp.sigpc)) + } + + // In case we are panicking from external C code + c.set_r10(uint32(uintptr(unsafe.Pointer(gp)))) + c.set_pc(uint32(funcPC(sigpanic))) + return + } + + if c.sigcode() == _SI_USER || flags&_SigNotify != 0 { + if sigsend(sig) { + return + } + } + + if flags&_SigKill != 0 { + exit(2) + } + + if flags&_SigThrow == 0 { + return + } + + _g_.m.throwing = 1 + _g_.m.caughtsig = gp + startpanic() + + if sig < uint32(len(sigtable)) { + print(sigtable[sig].name, "\n") + } else { + print("Signal ", sig, "\n") + } + + print("PC=", hex(c.pc()), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + goroutineheader(gp) + tracebacktrap(uintptr(c.pc()), uintptr(c.sp()), uintptr(c.lr()), gp) + tracebackothers(gp) + print("\n") + dumpregs(c) + } + + if docrash { + crash() + } + + exit(2) +} diff --git a/src/runtime/signal_darwin.go b/src/runtime/signal_darwin.go new file mode 100644 index 000000000..122648bc3 --- /dev/null +++ b/src/runtime/signal_darwin.go @@ -0,0 +1,45 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: abort"}, + /* 7 */ {_SigThrow, "SIGEMT: emulate instruction executed"}, + /* 8 */ {_SigPanic, "SIGFPE: floating-point exception"}, + /* 9 */ {0, "SIGKILL: kill"}, + /* 10 */ {_SigPanic, "SIGBUS: bus error"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigThrow, "SIGSYS: bad system call"}, + /* 13 */ {_SigNotify, "SIGPIPE: write to broken pipe"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: termination"}, + /* 16 */ {_SigNotify, "SIGURG: urgent condition on socket"}, + /* 17 */ {0, "SIGSTOP: stop"}, + /* 18 */ {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + /* 19 */ {0, "SIGCONT: continue after stop"}, + /* 20 */ {_SigNotify, "SIGCHLD: child status has changed"}, + /* 21 */ {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + /* 22 */ {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + /* 23 */ {_SigNotify, "SIGIO: i/o now possible"}, + /* 24 */ {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + /* 25 */ {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + /* 26 */ {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + /* 27 */ {_SigNotify, "SIGPROF: profiling alarm clock"}, + /* 28 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 29 */ {_SigNotify, "SIGINFO: status request from keyboard"}, + /* 30 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, + /* 31 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, +} diff --git a/src/runtime/signal_darwin_386.go b/src/runtime/signal_darwin_386.go new file mode 100644 index 000000000..ccf30ef31 --- /dev/null +++ b/src/runtime/signal_darwin_386.go @@ -0,0 +1,34 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *regs32 { return &(*ucontext)(c.ctxt).uc_mcontext.ss } +func (c *sigctxt) eax() uint32 { return c.regs().eax } +func (c *sigctxt) ebx() uint32 { return c.regs().ebx } +func (c *sigctxt) ecx() uint32 { return c.regs().ecx } +func (c *sigctxt) edx() uint32 { return c.regs().edx } +func (c *sigctxt) edi() uint32 { return c.regs().edi } +func (c *sigctxt) esi() uint32 { return c.regs().esi } +func (c *sigctxt) ebp() uint32 { return c.regs().ebp } +func (c *sigctxt) esp() uint32 { return c.regs().esp } +func (c *sigctxt) eip() uint32 { return c.regs().eip } +func (c *sigctxt) eflags() uint32 { return c.regs().eflags } +func (c *sigctxt) cs() uint32 { return c.regs().cs } +func (c *sigctxt) fs() uint32 { return c.regs().fs } +func (c *sigctxt) gs() uint32 { return c.regs().gs } +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { return uint32(uintptr(unsafe.Pointer(c.info.si_addr))) } + +func (c *sigctxt) set_eip(x uint32) { c.regs().eip = x } +func (c *sigctxt) set_esp(x uint32) { c.regs().esp = x } +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { c.info.si_addr = (*byte)(unsafe.Pointer(uintptr(x))) } diff --git a/src/runtime/signal_darwin_386.h b/src/runtime/signal_darwin_386.h deleted file mode 100644 index 5459e10a1..000000000 --- a/src/runtime/signal_darwin_386.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext->ss) - -#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).eax) -#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).ebx) -#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).ecx) -#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).edx) -#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).edi) -#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).esi) -#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).ebp) -#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).esp) -#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).eip) -#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).eflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/runtime/signal_darwin_amd64.go b/src/runtime/signal_darwin_amd64.go new file mode 100644 index 000000000..409bc6d57 --- /dev/null +++ b/src/runtime/signal_darwin_amd64.go @@ -0,0 +1,42 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *regs64 { return &(*ucontext)(c.ctxt).uc_mcontext.ss } +func (c *sigctxt) rax() uint64 { return c.regs().rax } +func (c *sigctxt) rbx() uint64 { return c.regs().rbx } +func (c *sigctxt) rcx() uint64 { return c.regs().rcx } +func (c *sigctxt) rdx() uint64 { return c.regs().rdx } +func (c *sigctxt) rdi() uint64 { return c.regs().rdi } +func (c *sigctxt) rsi() uint64 { return c.regs().rsi } +func (c *sigctxt) rbp() uint64 { return c.regs().rbp } +func (c *sigctxt) rsp() uint64 { return c.regs().rsp } +func (c *sigctxt) r8() uint64 { return c.regs().r8 } +func (c *sigctxt) r9() uint64 { return c.regs().r9 } +func (c *sigctxt) r10() uint64 { return c.regs().r10 } +func (c *sigctxt) r11() uint64 { return c.regs().r11 } +func (c *sigctxt) r12() uint64 { return c.regs().r12 } +func (c *sigctxt) r13() uint64 { return c.regs().r13 } +func (c *sigctxt) r14() uint64 { return c.regs().r14 } +func (c *sigctxt) r15() uint64 { return c.regs().r15 } +func (c *sigctxt) rip() uint64 { return c.regs().rip } +func (c *sigctxt) rflags() uint64 { return c.regs().rflags } +func (c *sigctxt) cs() uint64 { return c.regs().cs } +func (c *sigctxt) fs() uint64 { return c.regs().fs } +func (c *sigctxt) gs() uint64 { return c.regs().gs } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return uint64(uintptr(unsafe.Pointer(c.info.si_addr))) } + +func (c *sigctxt) set_rip(x uint64) { c.regs().rip = x } +func (c *sigctxt) set_rsp(x uint64) { c.regs().rsp = x } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { c.info.si_addr = (*byte)(unsafe.Pointer(uintptr(x))) } diff --git a/src/runtime/signal_darwin_amd64.h b/src/runtime/signal_darwin_amd64.h deleted file mode 100644 index e3da6de3a..000000000 --- a/src/runtime/signal_darwin_amd64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext->ss) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).rax) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).rbx) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).rcx) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).rdx) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).rdi) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).rsi) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).rbp) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).rsp) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).r10) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).r11) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).r12) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).r13) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).r14) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).r15) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).rip) -#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).rflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/runtime/signal_dragonfly.go b/src/runtime/signal_dragonfly.go new file mode 100644 index 000000000..d37e11a3e --- /dev/null +++ b/src/runtime/signal_dragonfly.go @@ -0,0 +1,46 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: abort"}, + /* 7 */ {_SigThrow, "SIGEMT: emulate instruction executed"}, + /* 8 */ {_SigPanic, "SIGFPE: floating-point exception"}, + /* 9 */ {0, "SIGKILL: kill"}, + /* 10 */ {_SigPanic, "SIGBUS: bus error"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigThrow, "SIGSYS: bad system call"}, + /* 13 */ {_SigNotify, "SIGPIPE: write to broken pipe"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: termination"}, + /* 16 */ {_SigNotify, "SIGURG: urgent condition on socket"}, + /* 17 */ {0, "SIGSTOP: stop"}, + /* 18 */ {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + /* 19 */ {0, "SIGCONT: continue after stop"}, + /* 20 */ {_SigNotify, "SIGCHLD: child status has changed"}, + /* 21 */ {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + /* 22 */ {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + /* 23 */ {_SigNotify, "SIGIO: i/o now possible"}, + /* 24 */ {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + /* 25 */ {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + /* 26 */ {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + /* 27 */ {_SigNotify, "SIGPROF: profiling alarm clock"}, + /* 28 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 29 */ {_SigNotify, "SIGINFO: status request from keyboard"}, + /* 30 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, + /* 31 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, + /* 32 */ {_SigNotify, "SIGTHR: reserved"}, +} diff --git a/src/runtime/signal_dragonfly_amd64.go b/src/runtime/signal_dragonfly_amd64.go new file mode 100644 index 000000000..740959c58 --- /dev/null +++ b/src/runtime/signal_dragonfly_amd64.go @@ -0,0 +1,44 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *mcontext { + return (*mcontext)(unsafe.Pointer(&(*ucontext)(c.ctxt).uc_mcontext)) +} +func (c *sigctxt) rax() uint64 { return c.regs().mc_rax } +func (c *sigctxt) rbx() uint64 { return c.regs().mc_rbx } +func (c *sigctxt) rcx() uint64 { return c.regs().mc_rcx } +func (c *sigctxt) rdx() uint64 { return c.regs().mc_rdx } +func (c *sigctxt) rdi() uint64 { return c.regs().mc_rdi } +func (c *sigctxt) rsi() uint64 { return c.regs().mc_rsi } +func (c *sigctxt) rbp() uint64 { return c.regs().mc_rbp } +func (c *sigctxt) rsp() uint64 { return c.regs().mc_rsp } +func (c *sigctxt) r8() uint64 { return c.regs().mc_r8 } +func (c *sigctxt) r9() uint64 { return c.regs().mc_r9 } +func (c *sigctxt) r10() uint64 { return c.regs().mc_r10 } +func (c *sigctxt) r11() uint64 { return c.regs().mc_r11 } +func (c *sigctxt) r12() uint64 { return c.regs().mc_r12 } +func (c *sigctxt) r13() uint64 { return c.regs().mc_r13 } +func (c *sigctxt) r14() uint64 { return c.regs().mc_r14 } +func (c *sigctxt) r15() uint64 { return c.regs().mc_r15 } +func (c *sigctxt) rip() uint64 { return c.regs().mc_rip } +func (c *sigctxt) rflags() uint64 { return c.regs().mc_rflags } +func (c *sigctxt) cs() uint64 { return uint64(c.regs().mc_cs) } +func (c *sigctxt) fs() uint64 { return uint64(c.regs().mc_ss) } +func (c *sigctxt) gs() uint64 { return uint64(c.regs().mc_ss) } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return uint64(c.info.si_addr) } + +func (c *sigctxt) set_rip(x uint64) { c.regs().mc_rip = x } +func (c *sigctxt) set_rsp(x uint64) { c.regs().mc_rsp = x } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { c.info.si_addr = x } diff --git a/src/runtime/signal_dragonfly_amd64.h b/src/runtime/signal_dragonfly_amd64.h deleted file mode 100644 index 5b4f97782..000000000 --- a/src/runtime/signal_dragonfly_amd64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).mc_rax) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).mc_rbx) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).mc_rcx) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).mc_rdx) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).mc_rdi) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).mc_rsi) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).mc_rbp) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).mc_rsp) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).mc_r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).mc_r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).mc_r10) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).mc_r11) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).mc_r12) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).mc_r13) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).mc_r14) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).mc_r15) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).mc_rip) -#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).mc_rflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).mc_cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).mc_ss) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).mc_ss) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/runtime/signal_freebsd.go b/src/runtime/signal_freebsd.go new file mode 100644 index 000000000..1dbdb1bd9 --- /dev/null +++ b/src/runtime/signal_freebsd.go @@ -0,0 +1,46 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: abort"}, + /* 7 */ {_SigThrow, "SIGEMT: emulate instruction executed"}, + /* 8 */ {_SigPanic, "SIGFPE: floating-point exception"}, + /* 9 */ {0, "SIGKILL: kill"}, + /* 10 */ {_SigPanic, "SIGBUS: bus error"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigNotify, "SIGSYS: bad system call"}, + /* 13 */ {_SigNotify, "SIGPIPE: write to broken pipe"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: termination"}, + /* 16 */ {_SigNotify, "SIGURG: urgent condition on socket"}, + /* 17 */ {0, "SIGSTOP: stop"}, + /* 18 */ {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + /* 19 */ {0, "SIGCONT: continue after stop"}, + /* 20 */ {_SigNotify, "SIGCHLD: child status has changed"}, + /* 21 */ {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + /* 22 */ {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + /* 23 */ {_SigNotify, "SIGIO: i/o now possible"}, + /* 24 */ {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + /* 25 */ {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + /* 26 */ {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + /* 27 */ {_SigNotify, "SIGPROF: profiling alarm clock"}, + /* 28 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 29 */ {_SigNotify, "SIGINFO: status request from keyboard"}, + /* 30 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, + /* 31 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, + /* 32 */ {_SigNotify, "SIGTHR: reserved"}, +} diff --git a/src/runtime/signal_freebsd_386.go b/src/runtime/signal_freebsd_386.go new file mode 100644 index 000000000..a0fec1309 --- /dev/null +++ b/src/runtime/signal_freebsd_386.go @@ -0,0 +1,34 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *mcontext { return &(*ucontext)(c.ctxt).uc_mcontext } +func (c *sigctxt) eax() uint32 { return c.regs().mc_eax } +func (c *sigctxt) ebx() uint32 { return c.regs().mc_ebx } +func (c *sigctxt) ecx() uint32 { return c.regs().mc_ecx } +func (c *sigctxt) edx() uint32 { return c.regs().mc_edx } +func (c *sigctxt) edi() uint32 { return c.regs().mc_edi } +func (c *sigctxt) esi() uint32 { return c.regs().mc_esi } +func (c *sigctxt) ebp() uint32 { return c.regs().mc_ebp } +func (c *sigctxt) esp() uint32 { return c.regs().mc_esp } +func (c *sigctxt) eip() uint32 { return c.regs().mc_eip } +func (c *sigctxt) eflags() uint32 { return c.regs().mc_eflags } +func (c *sigctxt) cs() uint32 { return uint32(c.regs().mc_cs) } +func (c *sigctxt) fs() uint32 { return uint32(c.regs().mc_fs) } +func (c *sigctxt) gs() uint32 { return uint32(c.regs().mc_gs) } +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { return uint32(c.info.si_addr) } + +func (c *sigctxt) set_eip(x uint32) { c.regs().mc_eip = x } +func (c *sigctxt) set_esp(x uint32) { c.regs().mc_esp = x } +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { c.info.si_addr = uintptr(x) } diff --git a/src/runtime/signal_freebsd_386.h b/src/runtime/signal_freebsd_386.h deleted file mode 100644 index a24f1ee96..000000000 --- a/src/runtime/signal_freebsd_386.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) - -#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).mc_eax) -#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).mc_ebx) -#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).mc_ecx) -#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).mc_edx) -#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).mc_edi) -#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).mc_esi) -#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).mc_ebp) -#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).mc_esp) -#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).mc_eip) -#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).mc_eflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).mc_cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).mc_fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).mc_gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/runtime/signal_freebsd_amd64.go b/src/runtime/signal_freebsd_amd64.go new file mode 100644 index 000000000..d10c883d4 --- /dev/null +++ b/src/runtime/signal_freebsd_amd64.go @@ -0,0 +1,44 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *mcontext { + return (*mcontext)(unsafe.Pointer(&(*ucontext)(c.ctxt).uc_mcontext)) +} +func (c *sigctxt) rax() uint64 { return c.regs().mc_rax } +func (c *sigctxt) rbx() uint64 { return c.regs().mc_rbx } +func (c *sigctxt) rcx() uint64 { return c.regs().mc_rcx } +func (c *sigctxt) rdx() uint64 { return c.regs().mc_rdx } +func (c *sigctxt) rdi() uint64 { return c.regs().mc_rdi } +func (c *sigctxt) rsi() uint64 { return c.regs().mc_rsi } +func (c *sigctxt) rbp() uint64 { return c.regs().mc_rbp } +func (c *sigctxt) rsp() uint64 { return c.regs().mc_rsp } +func (c *sigctxt) r8() uint64 { return c.regs().mc_r8 } +func (c *sigctxt) r9() uint64 { return c.regs().mc_r9 } +func (c *sigctxt) r10() uint64 { return c.regs().mc_r10 } +func (c *sigctxt) r11() uint64 { return c.regs().mc_r11 } +func (c *sigctxt) r12() uint64 { return c.regs().mc_r12 } +func (c *sigctxt) r13() uint64 { return c.regs().mc_r13 } +func (c *sigctxt) r14() uint64 { return c.regs().mc_r14 } +func (c *sigctxt) r15() uint64 { return c.regs().mc_r15 } +func (c *sigctxt) rip() uint64 { return c.regs().mc_rip } +func (c *sigctxt) rflags() uint64 { return c.regs().mc_rflags } +func (c *sigctxt) cs() uint64 { return uint64(c.regs().mc_cs) } +func (c *sigctxt) fs() uint64 { return uint64(c.regs().mc_fs) } +func (c *sigctxt) gs() uint64 { return uint64(c.regs().mc_gs) } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return uint64(c.info.si_addr) } + +func (c *sigctxt) set_rip(x uint64) { c.regs().mc_rip = x } +func (c *sigctxt) set_rsp(x uint64) { c.regs().mc_rsp = x } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { c.info.si_addr = x } diff --git a/src/runtime/signal_freebsd_amd64.h b/src/runtime/signal_freebsd_amd64.h deleted file mode 100644 index 7d35b7f85..000000000 --- a/src/runtime/signal_freebsd_amd64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).mc_rax) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).mc_rbx) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).mc_rcx) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).mc_rdx) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).mc_rdi) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).mc_rsi) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).mc_rbp) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).mc_rsp) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).mc_r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).mc_r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).mc_r10) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).mc_r11) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).mc_r12) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).mc_r13) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).mc_r14) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).mc_r15) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).mc_rip) -#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).mc_rflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).mc_cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).mc_fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).mc_gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/runtime/signal_freebsd_arm.go b/src/runtime/signal_freebsd_arm.go new file mode 100644 index 000000000..12de23d58 --- /dev/null +++ b/src/runtime/signal_freebsd_arm.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *mcontext { return &(*ucontext)(c.ctxt).uc_mcontext } +func (c *sigctxt) r0() uint32 { return c.regs().__gregs[0] } +func (c *sigctxt) r1() uint32 { return c.regs().__gregs[1] } +func (c *sigctxt) r2() uint32 { return c.regs().__gregs[2] } +func (c *sigctxt) r3() uint32 { return c.regs().__gregs[3] } +func (c *sigctxt) r4() uint32 { return c.regs().__gregs[4] } +func (c *sigctxt) r5() uint32 { return c.regs().__gregs[5] } +func (c *sigctxt) r6() uint32 { return c.regs().__gregs[6] } +func (c *sigctxt) r7() uint32 { return c.regs().__gregs[7] } +func (c *sigctxt) r8() uint32 { return c.regs().__gregs[8] } +func (c *sigctxt) r9() uint32 { return c.regs().__gregs[9] } +func (c *sigctxt) r10() uint32 { return c.regs().__gregs[10] } +func (c *sigctxt) fp() uint32 { return c.regs().__gregs[11] } +func (c *sigctxt) ip() uint32 { return c.regs().__gregs[12] } +func (c *sigctxt) sp() uint32 { return c.regs().__gregs[13] } +func (c *sigctxt) lr() uint32 { return c.regs().__gregs[14] } +func (c *sigctxt) pc() uint32 { return c.regs().__gregs[15] } +func (c *sigctxt) cpsr() uint32 { return c.regs().__gregs[16] } +func (c *sigctxt) fault() uint32 { return uint32(c.info.si_addr) } +func (c *sigctxt) trap() uint32 { return 0 } +func (c *sigctxt) error() uint32 { return 0 } +func (c *sigctxt) oldmask() uint32 { return 0 } + +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { return uint32(c.info.si_addr) } + +func (c *sigctxt) set_pc(x uint32) { c.regs().__gregs[15] = x } +func (c *sigctxt) set_sp(x uint32) { c.regs().__gregs[13] = x } +func (c *sigctxt) set_lr(x uint32) { c.regs().__gregs[14] = x } +func (c *sigctxt) set_r10(x uint32) { c.regs().__gregs[10] = x } + +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { + c.info.si_addr = uintptr(x) +} diff --git a/src/runtime/signal_freebsd_arm.h b/src/runtime/signal_freebsd_arm.h deleted file mode 100644 index 87a45aa27..000000000 --- a/src/runtime/signal_freebsd_arm.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) - -#define SIG_R0(info, ctxt) (SIG_REGS(ctxt).__gregs[0]) -#define SIG_R1(info, ctxt) (SIG_REGS(ctxt).__gregs[1]) -#define SIG_R2(info, ctxt) (SIG_REGS(ctxt).__gregs[2]) -#define SIG_R3(info, ctxt) (SIG_REGS(ctxt).__gregs[3]) -#define SIG_R4(info, ctxt) (SIG_REGS(ctxt).__gregs[4]) -#define SIG_R5(info, ctxt) (SIG_REGS(ctxt).__gregs[5]) -#define SIG_R6(info, ctxt) (SIG_REGS(ctxt).__gregs[6]) -#define SIG_R7(info, ctxt) (SIG_REGS(ctxt).__gregs[7]) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).__gregs[8]) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).__gregs[9]) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).__gregs[10]) -#define SIG_FP(info, ctxt) (SIG_REGS(ctxt).__gregs[11]) -#define SIG_IP(info, ctxt) (SIG_REGS(ctxt).__gregs[12]) -#define SIG_SP(info, ctxt) (SIG_REGS(ctxt).__gregs[13]) -#define SIG_LR(info, ctxt) (SIG_REGS(ctxt).__gregs[14]) -#define SIG_PC(info, ctxt) (SIG_REGS(ctxt).__gregs[15]) -#define SIG_CPSR(info, ctxt) (SIG_REGS(ctxt).__gregs[16]) -#define SIG_FAULT(info, ctxt) ((uintptr)(info)->si_addr) -#define SIG_TRAP(info, ctxt) (0) -#define SIG_ERROR(info, ctxt) (0) -#define SIG_OLDMASK(info, ctxt) (0) -#define SIG_CODE0(info, ctxt) ((uintptr)(info)->si_code) diff --git a/src/runtime/signal_linux.go b/src/runtime/signal_linux.go new file mode 100644 index 000000000..1c3d6872b --- /dev/null +++ b/src/runtime/signal_linux.go @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: abort"}, + /* 7 */ {_SigPanic, "SIGBUS: bus error"}, + /* 8 */ {_SigPanic, "SIGFPE: floating-point exception"}, + /* 9 */ {0, "SIGKILL: kill"}, + /* 10 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, + /* 13 */ {_SigNotify, "SIGPIPE: write to broken pipe"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: termination"}, + /* 16 */ {_SigThrow, "SIGSTKFLT: stack fault"}, + /* 17 */ {_SigNotify, "SIGCHLD: child status has changed"}, + /* 18 */ {0, "SIGCONT: continue"}, + /* 19 */ {0, "SIGSTOP: stop, unblockable"}, + /* 20 */ {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + /* 21 */ {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + /* 22 */ {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + /* 23 */ {_SigNotify, "SIGURG: urgent condition on socket"}, + /* 24 */ {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + /* 25 */ {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + /* 26 */ {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + /* 27 */ {_SigNotify, "SIGPROF: profiling alarm clock"}, + /* 28 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 29 */ {_SigNotify, "SIGIO: i/o now possible"}, + /* 30 */ {_SigNotify, "SIGPWR: power failure restart"}, + /* 31 */ {_SigNotify, "SIGSYS: bad system call"}, + /* 32 */ {0, "signal 32"}, /* SIGCANCEL; see issue 6997 */ + /* 33 */ {0, "signal 33"}, /* SIGSETXID; see issue 3871 */ + /* 34 */ {_SigNotify, "signal 34"}, + /* 35 */ {_SigNotify, "signal 35"}, + /* 36 */ {_SigNotify, "signal 36"}, + /* 37 */ {_SigNotify, "signal 37"}, + /* 38 */ {_SigNotify, "signal 38"}, + /* 39 */ {_SigNotify, "signal 39"}, + /* 40 */ {_SigNotify, "signal 40"}, + /* 41 */ {_SigNotify, "signal 41"}, + /* 42 */ {_SigNotify, "signal 42"}, + /* 43 */ {_SigNotify, "signal 43"}, + /* 44 */ {_SigNotify, "signal 44"}, + /* 45 */ {_SigNotify, "signal 45"}, + /* 46 */ {_SigNotify, "signal 46"}, + /* 47 */ {_SigNotify, "signal 47"}, + /* 48 */ {_SigNotify, "signal 48"}, + /* 49 */ {_SigNotify, "signal 49"}, + /* 50 */ {_SigNotify, "signal 50"}, + /* 51 */ {_SigNotify, "signal 51"}, + /* 52 */ {_SigNotify, "signal 52"}, + /* 53 */ {_SigNotify, "signal 53"}, + /* 54 */ {_SigNotify, "signal 54"}, + /* 55 */ {_SigNotify, "signal 55"}, + /* 56 */ {_SigNotify, "signal 56"}, + /* 57 */ {_SigNotify, "signal 57"}, + /* 58 */ {_SigNotify, "signal 58"}, + /* 59 */ {_SigNotify, "signal 59"}, + /* 60 */ {_SigNotify, "signal 60"}, + /* 61 */ {_SigNotify, "signal 61"}, + /* 62 */ {_SigNotify, "signal 62"}, + /* 63 */ {_SigNotify, "signal 63"}, + /* 64 */ {_SigNotify, "signal 64"}, +} diff --git a/src/runtime/signal_linux_386.go b/src/runtime/signal_linux_386.go new file mode 100644 index 000000000..085f66e89 --- /dev/null +++ b/src/runtime/signal_linux_386.go @@ -0,0 +1,36 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *sigcontext { return &(*ucontext)(c.ctxt).uc_mcontext } +func (c *sigctxt) eax() uint32 { return c.regs().eax } +func (c *sigctxt) ebx() uint32 { return c.regs().ebx } +func (c *sigctxt) ecx() uint32 { return c.regs().ecx } +func (c *sigctxt) edx() uint32 { return c.regs().edx } +func (c *sigctxt) edi() uint32 { return c.regs().edi } +func (c *sigctxt) esi() uint32 { return c.regs().esi } +func (c *sigctxt) ebp() uint32 { return c.regs().ebp } +func (c *sigctxt) esp() uint32 { return c.regs().esp } +func (c *sigctxt) eip() uint32 { return c.regs().eip } +func (c *sigctxt) eflags() uint32 { return c.regs().eflags } +func (c *sigctxt) cs() uint32 { return uint32(c.regs().cs) } +func (c *sigctxt) fs() uint32 { return uint32(c.regs().fs) } +func (c *sigctxt) gs() uint32 { return uint32(c.regs().gs) } +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { return c.info.si_addr } + +func (c *sigctxt) set_eip(x uint32) { c.regs().eip = x } +func (c *sigctxt) set_esp(x uint32) { c.regs().esp = x } +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { + *(*uintptr)(add(unsafe.Pointer(c.info), 2*ptrSize)) = uintptr(x) +} diff --git a/src/runtime/signal_linux_386.h b/src/runtime/signal_linux_386.h deleted file mode 100644 index f77f1c9d5..000000000 --- a/src/runtime/signal_linux_386.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) - -#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).eax) -#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).ebx) -#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).ecx) -#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).edx) -#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).edi) -#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).esi) -#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).ebp) -#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).esp) -#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).eip) -#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).eflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) (((uintptr*)(info))[2]) - diff --git a/src/runtime/signal_linux_amd64.go b/src/runtime/signal_linux_amd64.go new file mode 100644 index 000000000..5e339b8a4 --- /dev/null +++ b/src/runtime/signal_linux_amd64.go @@ -0,0 +1,46 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *sigcontext { + return (*sigcontext)(unsafe.Pointer(&(*ucontext)(c.ctxt).uc_mcontext)) +} +func (c *sigctxt) rax() uint64 { return c.regs().rax } +func (c *sigctxt) rbx() uint64 { return c.regs().rbx } +func (c *sigctxt) rcx() uint64 { return c.regs().rcx } +func (c *sigctxt) rdx() uint64 { return c.regs().rdx } +func (c *sigctxt) rdi() uint64 { return c.regs().rdi } +func (c *sigctxt) rsi() uint64 { return c.regs().rsi } +func (c *sigctxt) rbp() uint64 { return c.regs().rbp } +func (c *sigctxt) rsp() uint64 { return c.regs().rsp } +func (c *sigctxt) r8() uint64 { return c.regs().r8 } +func (c *sigctxt) r9() uint64 { return c.regs().r9 } +func (c *sigctxt) r10() uint64 { return c.regs().r10 } +func (c *sigctxt) r11() uint64 { return c.regs().r11 } +func (c *sigctxt) r12() uint64 { return c.regs().r12 } +func (c *sigctxt) r13() uint64 { return c.regs().r13 } +func (c *sigctxt) r14() uint64 { return c.regs().r14 } +func (c *sigctxt) r15() uint64 { return c.regs().r15 } +func (c *sigctxt) rip() uint64 { return c.regs().rip } +func (c *sigctxt) rflags() uint64 { return c.regs().eflags } +func (c *sigctxt) cs() uint64 { return uint64(c.regs().cs) } +func (c *sigctxt) fs() uint64 { return uint64(c.regs().fs) } +func (c *sigctxt) gs() uint64 { return uint64(c.regs().gs) } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return c.info.si_addr } + +func (c *sigctxt) set_rip(x uint64) { c.regs().rip = x } +func (c *sigctxt) set_rsp(x uint64) { c.regs().rsp = x } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { + *(*uintptr)(add(unsafe.Pointer(c.info), 2*ptrSize)) = uintptr(x) +} diff --git a/src/runtime/signal_linux_amd64.h b/src/runtime/signal_linux_amd64.h deleted file mode 100644 index 5a9a3e5da..000000000 --- a/src/runtime/signal_linux_amd64.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).rax) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).rbx) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).rcx) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).rdx) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).rdi) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).rsi) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).rbp) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).rsp) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).r10) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).r11) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).r12) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).r13) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).r14) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).r15) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).rip) -#define SIG_RFLAGS(info, ctxt) ((uint64)SIG_REGS(ctxt).eflags) - -#define SIG_CS(info, ctxt) ((uint64)SIG_REGS(ctxt).cs) -#define SIG_FS(info, ctxt) ((uint64)SIG_REGS(ctxt).fs) -#define SIG_GS(info, ctxt) ((uint64)SIG_REGS(ctxt).gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) (((uintptr*)(info))[2]) - diff --git a/src/runtime/signal_linux_arm.go b/src/runtime/signal_linux_arm.go new file mode 100644 index 000000000..bdb4314fa --- /dev/null +++ b/src/runtime/signal_linux_arm.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *sigcontext { return &(*ucontext)(c.ctxt).uc_mcontext } +func (c *sigctxt) r0() uint32 { return c.regs().r0 } +func (c *sigctxt) r1() uint32 { return c.regs().r1 } +func (c *sigctxt) r2() uint32 { return c.regs().r2 } +func (c *sigctxt) r3() uint32 { return c.regs().r3 } +func (c *sigctxt) r4() uint32 { return c.regs().r4 } +func (c *sigctxt) r5() uint32 { return c.regs().r5 } +func (c *sigctxt) r6() uint32 { return c.regs().r6 } +func (c *sigctxt) r7() uint32 { return c.regs().r7 } +func (c *sigctxt) r8() uint32 { return c.regs().r8 } +func (c *sigctxt) r9() uint32 { return c.regs().r9 } +func (c *sigctxt) r10() uint32 { return c.regs().r10 } +func (c *sigctxt) fp() uint32 { return c.regs().fp } +func (c *sigctxt) ip() uint32 { return c.regs().ip } +func (c *sigctxt) sp() uint32 { return c.regs().sp } +func (c *sigctxt) lr() uint32 { return c.regs().lr } +func (c *sigctxt) pc() uint32 { return c.regs().pc } +func (c *sigctxt) cpsr() uint32 { return c.regs().cpsr } +func (c *sigctxt) fault() uint32 { return c.regs().fault_address } +func (c *sigctxt) trap() uint32 { return c.regs().trap_no } +func (c *sigctxt) error() uint32 { return c.regs().error_code } +func (c *sigctxt) oldmask() uint32 { return c.regs().oldmask } + +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { return c.info.si_addr } + +func (c *sigctxt) set_pc(x uint32) { c.regs().pc = x } +func (c *sigctxt) set_sp(x uint32) { c.regs().sp = x } +func (c *sigctxt) set_lr(x uint32) { c.regs().lr = x } +func (c *sigctxt) set_r10(x uint32) { c.regs().r10 = x } + +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { + *(*uintptr)(add(unsafe.Pointer(c.info), 2*ptrSize)) = uintptr(x) +} diff --git a/src/runtime/signal_linux_arm.h b/src/runtime/signal_linux_arm.h deleted file mode 100644 index a674c0d57..000000000 --- a/src/runtime/signal_linux_arm.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) - -#define SIG_R0(info, ctxt) (SIG_REGS(ctxt).arm_r0) -#define SIG_R1(info, ctxt) (SIG_REGS(ctxt).arm_r1) -#define SIG_R2(info, ctxt) (SIG_REGS(ctxt).arm_r2) -#define SIG_R3(info, ctxt) (SIG_REGS(ctxt).arm_r3) -#define SIG_R4(info, ctxt) (SIG_REGS(ctxt).arm_r4) -#define SIG_R5(info, ctxt) (SIG_REGS(ctxt).arm_r5) -#define SIG_R6(info, ctxt) (SIG_REGS(ctxt).arm_r6) -#define SIG_R7(info, ctxt) (SIG_REGS(ctxt).arm_r7) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).arm_r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).arm_r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).arm_r10) -#define SIG_FP(info, ctxt) (SIG_REGS(ctxt).arm_fp) -#define SIG_IP(info, ctxt) (SIG_REGS(ctxt).arm_ip) -#define SIG_SP(info, ctxt) (SIG_REGS(ctxt).arm_sp) -#define SIG_LR(info, ctxt) (SIG_REGS(ctxt).arm_lr) -#define SIG_PC(info, ctxt) (SIG_REGS(ctxt).arm_pc) -#define SIG_CPSR(info, ctxt) (SIG_REGS(ctxt).arm_cpsr) -#define SIG_FAULT(info, ctxt) (SIG_REGS(ctxt).fault_address) -#define SIG_TRAP(info, ctxt) (SIG_REGS(ctxt).trap_no) -#define SIG_ERROR(info, ctxt) (SIG_REGS(ctxt).error_code) -#define SIG_OLDMASK(info, ctxt) (SIG_REGS(ctxt).oldmask) -#define SIG_CODE0(info, ctxt) ((uintptr)(info)->si_code) diff --git a/src/runtime/signal_linux_power64x.go b/src/runtime/signal_linux_power64x.go new file mode 100644 index 000000000..0a406b31f --- /dev/null +++ b/src/runtime/signal_linux_power64x.go @@ -0,0 +1,71 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux +// +build power64 power64le + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *ptregs { return (*ucontext)(c.ctxt).uc_mcontext.regs } +func (c *sigctxt) r0() uint64 { return c.regs().gpr[0] } +func (c *sigctxt) r1() uint64 { return c.regs().gpr[1] } +func (c *sigctxt) r2() uint64 { return c.regs().gpr[2] } +func (c *sigctxt) r3() uint64 { return c.regs().gpr[3] } +func (c *sigctxt) r4() uint64 { return c.regs().gpr[4] } +func (c *sigctxt) r5() uint64 { return c.regs().gpr[5] } +func (c *sigctxt) r6() uint64 { return c.regs().gpr[6] } +func (c *sigctxt) r7() uint64 { return c.regs().gpr[7] } +func (c *sigctxt) r8() uint64 { return c.regs().gpr[8] } +func (c *sigctxt) r9() uint64 { return c.regs().gpr[9] } +func (c *sigctxt) r10() uint64 { return c.regs().gpr[10] } +func (c *sigctxt) r11() uint64 { return c.regs().gpr[11] } +func (c *sigctxt) r12() uint64 { return c.regs().gpr[12] } +func (c *sigctxt) r13() uint64 { return c.regs().gpr[13] } +func (c *sigctxt) r14() uint64 { return c.regs().gpr[14] } +func (c *sigctxt) r15() uint64 { return c.regs().gpr[15] } +func (c *sigctxt) r16() uint64 { return c.regs().gpr[16] } +func (c *sigctxt) r17() uint64 { return c.regs().gpr[17] } +func (c *sigctxt) r18() uint64 { return c.regs().gpr[18] } +func (c *sigctxt) r19() uint64 { return c.regs().gpr[19] } +func (c *sigctxt) r20() uint64 { return c.regs().gpr[20] } +func (c *sigctxt) r21() uint64 { return c.regs().gpr[21] } +func (c *sigctxt) r22() uint64 { return c.regs().gpr[22] } +func (c *sigctxt) r23() uint64 { return c.regs().gpr[23] } +func (c *sigctxt) r24() uint64 { return c.regs().gpr[24] } +func (c *sigctxt) r25() uint64 { return c.regs().gpr[25] } +func (c *sigctxt) r26() uint64 { return c.regs().gpr[26] } +func (c *sigctxt) r27() uint64 { return c.regs().gpr[27] } +func (c *sigctxt) r28() uint64 { return c.regs().gpr[28] } +func (c *sigctxt) r29() uint64 { return c.regs().gpr[29] } +func (c *sigctxt) r30() uint64 { return c.regs().gpr[30] } +func (c *sigctxt) r31() uint64 { return c.regs().gpr[31] } +func (c *sigctxt) sp() uint64 { return c.regs().gpr[1] } +func (c *sigctxt) pc() uint64 { return c.regs().nip } +func (c *sigctxt) trap() uint64 { return c.regs().trap } +func (c *sigctxt) ctr() uint64 { return c.regs().ctr } +func (c *sigctxt) link() uint64 { return c.regs().link } +func (c *sigctxt) xer() uint64 { return c.regs().xer } +func (c *sigctxt) ccr() uint64 { return c.regs().ccr } + +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return c.info.si_addr } +func (c *sigctxt) fault() uint64 { return c.regs().dar } + +func (c *sigctxt) set_r0(x uint64) { c.regs().gpr[0] = x } +func (c *sigctxt) set_r30(x uint64) { c.regs().gpr[30] = x } +func (c *sigctxt) set_pc(x uint64) { c.regs().nip = x } +func (c *sigctxt) set_sp(x uint64) { c.regs().gpr[1] = x } +func (c *sigctxt) set_link(x uint64) { c.regs().link = x } + +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { + *(*uintptr)(add(unsafe.Pointer(c.info), 2*ptrSize)) = uintptr(x) +} diff --git a/src/runtime/signal_openbsd.go b/src/runtime/signal_openbsd.go new file mode 100644 index 000000000..78afc59ef --- /dev/null +++ b/src/runtime/signal_openbsd.go @@ -0,0 +1,46 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: abort"}, + /* 7 */ {_SigThrow, "SIGEMT: emulate instruction executed"}, + /* 8 */ {_SigPanic, "SIGFPE: floating-point exception"}, + /* 9 */ {0, "SIGKILL: kill"}, + /* 10 */ {_SigPanic, "SIGBUS: bus error"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigThrow, "SIGSYS: bad system call"}, + /* 13 */ {_SigNotify, "SIGPIPE: write to broken pipe"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: termination"}, + /* 16 */ {_SigNotify, "SIGURG: urgent condition on socket"}, + /* 17 */ {0, "SIGSTOP: stop"}, + /* 18 */ {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + /* 19 */ {0, "SIGCONT: continue after stop"}, + /* 20 */ {_SigNotify, "SIGCHLD: child status has changed"}, + /* 21 */ {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + /* 22 */ {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + /* 23 */ {_SigNotify, "SIGIO: i/o now possible"}, + /* 24 */ {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + /* 25 */ {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + /* 26 */ {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + /* 27 */ {_SigNotify, "SIGPROF: profiling alarm clock"}, + /* 28 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 29 */ {_SigNotify, "SIGINFO: status request from keyboard"}, + /* 30 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, + /* 31 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, + /* 32 */ {_SigNotify, "SIGTHR: reserved"}, +} diff --git a/src/runtime/signal_openbsd_386.go b/src/runtime/signal_openbsd_386.go new file mode 100644 index 000000000..c582a4493 --- /dev/null +++ b/src/runtime/signal_openbsd_386.go @@ -0,0 +1,41 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *sigcontext { + return (*sigcontext)(c.ctxt) +} + +func (c *sigctxt) eax() uint32 { return c.regs().sc_eax } +func (c *sigctxt) ebx() uint32 { return c.regs().sc_ebx } +func (c *sigctxt) ecx() uint32 { return c.regs().sc_ecx } +func (c *sigctxt) edx() uint32 { return c.regs().sc_edx } +func (c *sigctxt) edi() uint32 { return c.regs().sc_edi } +func (c *sigctxt) esi() uint32 { return c.regs().sc_esi } +func (c *sigctxt) ebp() uint32 { return c.regs().sc_ebp } +func (c *sigctxt) esp() uint32 { return c.regs().sc_esp } +func (c *sigctxt) eip() uint32 { return c.regs().sc_eip } +func (c *sigctxt) eflags() uint32 { return c.regs().sc_eflags } +func (c *sigctxt) cs() uint32 { return c.regs().sc_cs } +func (c *sigctxt) fs() uint32 { return c.regs().sc_fs } +func (c *sigctxt) gs() uint32 { return c.regs().sc_gs } +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint32 { + return *(*uint32)(add(unsafe.Pointer(c.info), 12)) +} + +func (c *sigctxt) set_eip(x uint32) { c.regs().sc_eip = x } +func (c *sigctxt) set_esp(x uint32) { c.regs().sc_esp = x } +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint32) { + *(*uint32)(add(unsafe.Pointer(c.info), 12)) = x +} diff --git a/src/runtime/signal_openbsd_386.h b/src/runtime/signal_openbsd_386.h deleted file mode 100644 index 6742db8d4..000000000 --- a/src/runtime/signal_openbsd_386.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (*(Sigcontext*)(ctxt)) - -#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).sc_eax) -#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).sc_ebx) -#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).sc_ecx) -#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).sc_edx) -#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).sc_edi) -#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).sc_esi) -#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).sc_ebp) -#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).sc_esp) -#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).sc_eip) -#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).sc_eflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).sc_cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).sc_fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).sc_gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) (*(uintptr*)((byte*)info + 12)) diff --git a/src/runtime/signal_openbsd_amd64.go b/src/runtime/signal_openbsd_amd64.go new file mode 100644 index 000000000..4f0d19ddd --- /dev/null +++ b/src/runtime/signal_openbsd_amd64.go @@ -0,0 +1,49 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *sigcontext { + return (*sigcontext)(c.ctxt) +} + +func (c *sigctxt) rax() uint64 { return c.regs().sc_rax } +func (c *sigctxt) rbx() uint64 { return c.regs().sc_rbx } +func (c *sigctxt) rcx() uint64 { return c.regs().sc_rcx } +func (c *sigctxt) rdx() uint64 { return c.regs().sc_rdx } +func (c *sigctxt) rdi() uint64 { return c.regs().sc_rdi } +func (c *sigctxt) rsi() uint64 { return c.regs().sc_rsi } +func (c *sigctxt) rbp() uint64 { return c.regs().sc_rbp } +func (c *sigctxt) rsp() uint64 { return c.regs().sc_rsp } +func (c *sigctxt) r8() uint64 { return c.regs().sc_r8 } +func (c *sigctxt) r9() uint64 { return c.regs().sc_r9 } +func (c *sigctxt) r10() uint64 { return c.regs().sc_r10 } +func (c *sigctxt) r11() uint64 { return c.regs().sc_r11 } +func (c *sigctxt) r12() uint64 { return c.regs().sc_r12 } +func (c *sigctxt) r13() uint64 { return c.regs().sc_r13 } +func (c *sigctxt) r14() uint64 { return c.regs().sc_r14 } +func (c *sigctxt) r15() uint64 { return c.regs().sc_r15 } +func (c *sigctxt) rip() uint64 { return c.regs().sc_rip } +func (c *sigctxt) rflags() uint64 { return c.regs().sc_rflags } +func (c *sigctxt) cs() uint64 { return c.regs().sc_cs } +func (c *sigctxt) fs() uint64 { return c.regs().sc_fs } +func (c *sigctxt) gs() uint64 { return c.regs().sc_gs } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { + return *(*uint64)(add(unsafe.Pointer(c.info), 16)) +} + +func (c *sigctxt) set_rip(x uint64) { c.regs().sc_rip = x } +func (c *sigctxt) set_rsp(x uint64) { c.regs().sc_rsp = x } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { + *(*uint64)(add(unsafe.Pointer(c.info), 16)) = x +} diff --git a/src/runtime/signal_openbsd_amd64.h b/src/runtime/signal_openbsd_amd64.h deleted file mode 100644 index b46a5dfa6..000000000 --- a/src/runtime/signal_openbsd_amd64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (*(Sigcontext*)(ctxt)) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).sc_rax) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).sc_rbx) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).sc_rcx) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).sc_rdx) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).sc_rdi) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).sc_rsi) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).sc_rbp) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).sc_rsp) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).sc_r8) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).sc_r9) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).sc_r10) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).sc_r11) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).sc_r12) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).sc_r13) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).sc_r14) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).sc_r15) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).sc_rip) -#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).sc_rflags) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).sc_cs) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).sc_fs) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).sc_gs) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) (*(uintptr*)((byte*)(info) + 16)) diff --git a/src/runtime/signal_power64x.go b/src/runtime/signal_power64x.go new file mode 100644 index 000000000..fc83beb1b --- /dev/null +++ b/src/runtime/signal_power64x.go @@ -0,0 +1,144 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux +// +build power64 power64le + +package runtime + +import "unsafe" + +func dumpregs(c *sigctxt) { + print("r0 ", hex(c.r0()), "\t") + print("r1 ", hex(c.r1()), "\n") + print("r2 ", hex(c.r2()), "\t") + print("r3 ", hex(c.r3()), "\n") + print("r4 ", hex(c.r4()), "\t") + print("r5 ", hex(c.r5()), "\n") + print("r6 ", hex(c.r6()), "\t") + print("r7 ", hex(c.r7()), "\n") + print("r8 ", hex(c.r8()), "\t") + print("r9 ", hex(c.r9()), "\n") + print("r10 ", hex(c.r10()), "\t") + print("r11 ", hex(c.r11()), "\n") + print("r12 ", hex(c.r12()), "\t") + print("r13 ", hex(c.r13()), "\n") + print("r14 ", hex(c.r14()), "\t") + print("r15 ", hex(c.r15()), "\n") + print("r16 ", hex(c.r16()), "\t") + print("r17 ", hex(c.r17()), "\n") + print("r18 ", hex(c.r18()), "\t") + print("r19 ", hex(c.r19()), "\n") + print("r20 ", hex(c.r20()), "\t") + print("r21 ", hex(c.r21()), "\n") + print("r22 ", hex(c.r22()), "\t") + print("r23 ", hex(c.r23()), "\n") + print("r24 ", hex(c.r24()), "\t") + print("r25 ", hex(c.r25()), "\n") + print("r26 ", hex(c.r26()), "\t") + print("r27 ", hex(c.r27()), "\n") + print("r28 ", hex(c.r28()), "\t") + print("r29 ", hex(c.r29()), "\n") + print("r30 ", hex(c.r30()), "\t") + print("r31 ", hex(c.r31()), "\n") + print("pc ", hex(c.pc()), "\t") + print("ctr ", hex(c.ctr()), "\n") + print("link ", hex(c.link()), "\t") + print("xer ", hex(c.xer()), "\n") + print("ccr ", hex(c.ccr()), "\t") + print("trap ", hex(c.trap()), "\n") +} + +func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { + _g_ := getg() + c := &sigctxt{info, ctxt} + + if sig == _SIGPROF { + sigprof((*byte)(unsafe.Pointer(uintptr(c.pc()))), (*byte)(unsafe.Pointer(uintptr(c.sp()))), (*byte)(unsafe.Pointer(uintptr(c.link()))), gp, _g_.m) + return + } + flags := int32(_SigThrow) + if sig < uint32(len(sigtable)) { + flags = sigtable[sig].flags + } + if c.sigcode() != _SI_USER && flags&_SigPanic != 0 { + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp.sig = sig + gp.sigcode0 = uintptr(c.sigcode()) + gp.sigcode1 = uintptr(c.fault()) + gp.sigpc = uintptr(c.pc()) + + // We arrange link, and pc to pretend the panicking + // function calls sigpanic directly. + // Always save LINK to stack so that panics in leaf + // functions are correctly handled. This smashes + // the stack frame but we're not going back there + // anyway. + sp := c.sp() - ptrSize + c.set_sp(sp) + *(*uint64)(unsafe.Pointer(uintptr(sp))) = c.link() + + // Don't bother saving PC if it's zero, which is + // probably a call to a nil func: the old link register + // is more useful in the stack trace. + if gp.sigpc != 0 { + c.set_link(uint64(gp.sigpc)) + } + + // In case we are panicking from external C code + c.set_r0(0) + c.set_r30(uint64(uintptr(unsafe.Pointer(gp)))) + c.set_pc(uint64(funcPC(sigpanic))) + return + } + + if c.sigcode() == _SI_USER || flags&_SigNotify != 0 { + if sigsend(sig) { + return + } + } + + if flags&_SigKill != 0 { + exit(2) + } + + if flags&_SigThrow == 0 { + return + } + + _g_.m.throwing = 1 + _g_.m.caughtsig = gp + startpanic() + + if sig < uint32(len(sigtable)) { + print(sigtable[sig].name, "\n") + } else { + print("Signal ", sig, "\n") + } + + print("PC=", hex(c.pc()), "\n") + if _g_.m.lockedg != nil && _g_.m.ncgo > 0 && gp == _g_.m.g0 { + print("signal arrived during cgo execution\n") + gp = _g_.m.lockedg + } + print("\n") + + var docrash bool + if gotraceback(&docrash) > 0 { + goroutineheader(gp) + tracebacktrap(uintptr(c.pc()), uintptr(c.sp()), uintptr(c.link()), gp) + tracebackothers(gp) + print("\n") + dumpregs(c) + } + + if docrash { + crash() + } + + exit(2) +} diff --git a/src/runtime/signal_solaris.go b/src/runtime/signal_solaris.go new file mode 100644 index 000000000..2986c5aab --- /dev/null +++ b/src/runtime/signal_solaris.go @@ -0,0 +1,88 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type sigTabT struct { + flags int32 + name string +} + +var sigtable = [...]sigTabT{ + /* 0 */ {0, "SIGNONE: no trap"}, + /* 1 */ {_SigNotify + _SigKill, "SIGHUP: hangup"}, + /* 2 */ {_SigNotify + _SigKill, "SIGINT: interrupt (rubout)"}, + /* 3 */ {_SigNotify + _SigThrow, "SIGQUIT: quit (ASCII FS)"}, + /* 4 */ {_SigThrow, "SIGILL: illegal instruction (not reset when caught)"}, + /* 5 */ {_SigThrow, "SIGTRAP: trace trap (not reset when caught)"}, + /* 6 */ {_SigNotify + _SigThrow, "SIGABRT: used by abort, replace SIGIOT in the future"}, + /* 7 */ {_SigThrow, "SIGEMT: EMT instruction"}, + /* 8 */ {_SigPanic, "SIGFPE: floating point exception"}, + /* 9 */ {0, "SIGKILL: kill (cannot be caught or ignored)"}, + /* 10 */ {_SigPanic, "SIGBUS: bus error"}, + /* 11 */ {_SigPanic, "SIGSEGV: segmentation violation"}, + /* 12 */ {_SigThrow, "SIGSYS: bad argument to system call"}, + /* 13 */ {_SigNotify, "SIGPIPE: write on a pipe with no one to read it"}, + /* 14 */ {_SigNotify, "SIGALRM: alarm clock"}, + /* 15 */ {_SigNotify + _SigKill, "SIGTERM: software termination signal from kill"}, + /* 16 */ {_SigNotify, "SIGUSR1: user defined signal 1"}, + /* 17 */ {_SigNotify, "SIGUSR2: user defined signal 2"}, + /* 18 */ {_SigNotify, "SIGCHLD: child status change alias (POSIX)"}, + /* 19 */ {_SigNotify, "SIGPWR: power-fail restart"}, + /* 20 */ {_SigNotify, "SIGWINCH: window size change"}, + /* 21 */ {_SigNotify, "SIGURG: urgent socket condition"}, + /* 22 */ {_SigNotify, "SIGPOLL: pollable event occured"}, + /* 23 */ {_SigNotify + _SigDefault, "SIGSTOP: stop (cannot be caught or ignored)"}, + /* 24 */ {0, "SIGTSTP: user stop requested from tty"}, + /* 25 */ {0, "SIGCONT: stopped process has been continued"}, + /* 26 */ {_SigNotify + _SigDefault, "SIGTTIN: background tty read attempted"}, + /* 27 */ {_SigNotify + _SigDefault, "SIGTTOU: background tty write attempted"}, + /* 28 */ {_SigNotify, "SIGVTALRM: virtual timer expired"}, + /* 29 */ {_SigNotify, "SIGPROF: profiling timer expired"}, + /* 30 */ {_SigNotify, "SIGXCPU: exceeded cpu limit"}, + /* 31 */ {_SigNotify, "SIGXFSZ: exceeded file size limit"}, + /* 32 */ {_SigNotify, "SIGWAITING: reserved signal no longer used by"}, + /* 33 */ {_SigNotify, "SIGLWP: reserved signal no longer used by"}, + /* 34 */ {_SigNotify, "SIGFREEZE: special signal used by CPR"}, + /* 35 */ {_SigNotify, "SIGTHAW: special signal used by CPR"}, + /* 36 */ {0, "SIGCANCEL: reserved signal for thread cancellation"}, + /* 37 */ {_SigNotify, "SIGLOST: resource lost (eg, record-lock lost)"}, + /* 38 */ {_SigNotify, "SIGXRES: resource control exceeded"}, + /* 39 */ {_SigNotify, "SIGJVM1: reserved signal for Java Virtual Machine"}, + /* 40 */ {_SigNotify, "SIGJVM2: reserved signal for Java Virtual Machine"}, + + /* TODO(aram): what should be do about these signals? _SigDefault or _SigNotify? is this set static? */ + /* 41 */ {_SigNotify, "real time signal"}, + /* 42 */ {_SigNotify, "real time signal"}, + /* 43 */ {_SigNotify, "real time signal"}, + /* 44 */ {_SigNotify, "real time signal"}, + /* 45 */ {_SigNotify, "real time signal"}, + /* 46 */ {_SigNotify, "real time signal"}, + /* 47 */ {_SigNotify, "real time signal"}, + /* 48 */ {_SigNotify, "real time signal"}, + /* 49 */ {_SigNotify, "real time signal"}, + /* 50 */ {_SigNotify, "real time signal"}, + /* 51 */ {_SigNotify, "real time signal"}, + /* 52 */ {_SigNotify, "real time signal"}, + /* 53 */ {_SigNotify, "real time signal"}, + /* 54 */ {_SigNotify, "real time signal"}, + /* 55 */ {_SigNotify, "real time signal"}, + /* 56 */ {_SigNotify, "real time signal"}, + /* 57 */ {_SigNotify, "real time signal"}, + /* 58 */ {_SigNotify, "real time signal"}, + /* 59 */ {_SigNotify, "real time signal"}, + /* 60 */ {_SigNotify, "real time signal"}, + /* 61 */ {_SigNotify, "real time signal"}, + /* 62 */ {_SigNotify, "real time signal"}, + /* 63 */ {_SigNotify, "real time signal"}, + /* 64 */ {_SigNotify, "real time signal"}, + /* 65 */ {_SigNotify, "real time signal"}, + /* 66 */ {_SigNotify, "real time signal"}, + /* 67 */ {_SigNotify, "real time signal"}, + /* 68 */ {_SigNotify, "real time signal"}, + /* 69 */ {_SigNotify, "real time signal"}, + /* 70 */ {_SigNotify, "real time signal"}, + /* 71 */ {_SigNotify, "real time signal"}, + /* 72 */ {_SigNotify, "real time signal"}, +} diff --git a/src/runtime/signal_solaris_amd64.go b/src/runtime/signal_solaris_amd64.go new file mode 100644 index 000000000..a577c8c19 --- /dev/null +++ b/src/runtime/signal_solaris_amd64.go @@ -0,0 +1,46 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +func (c *sigctxt) regs() *mcontext { + return (*mcontext)(unsafe.Pointer(&(*ucontext)(c.ctxt).uc_mcontext)) +} +func (c *sigctxt) rax() uint64 { return uint64(c.regs().gregs[_REG_RAX]) } +func (c *sigctxt) rbx() uint64 { return uint64(c.regs().gregs[_REG_RBX]) } +func (c *sigctxt) rcx() uint64 { return uint64(c.regs().gregs[_REG_RCX]) } +func (c *sigctxt) rdx() uint64 { return uint64(c.regs().gregs[_REG_RDX]) } +func (c *sigctxt) rdi() uint64 { return uint64(c.regs().gregs[_REG_RDI]) } +func (c *sigctxt) rsi() uint64 { return uint64(c.regs().gregs[_REG_RSI]) } +func (c *sigctxt) rbp() uint64 { return uint64(c.regs().gregs[_REG_RBP]) } +func (c *sigctxt) rsp() uint64 { return uint64(c.regs().gregs[_REG_RSP]) } +func (c *sigctxt) r8() uint64 { return uint64(c.regs().gregs[_REG_R8]) } +func (c *sigctxt) r9() uint64 { return uint64(c.regs().gregs[_REG_R9]) } +func (c *sigctxt) r10() uint64 { return uint64(c.regs().gregs[_REG_R10]) } +func (c *sigctxt) r11() uint64 { return uint64(c.regs().gregs[_REG_R11]) } +func (c *sigctxt) r12() uint64 { return uint64(c.regs().gregs[_REG_R12]) } +func (c *sigctxt) r13() uint64 { return uint64(c.regs().gregs[_REG_R13]) } +func (c *sigctxt) r14() uint64 { return uint64(c.regs().gregs[_REG_R14]) } +func (c *sigctxt) r15() uint64 { return uint64(c.regs().gregs[_REG_R15]) } +func (c *sigctxt) rip() uint64 { return uint64(c.regs().gregs[_REG_RIP]) } +func (c *sigctxt) rflags() uint64 { return uint64(c.regs().gregs[_REG_RFLAGS]) } +func (c *sigctxt) cs() uint64 { return uint64(c.regs().gregs[_REG_CS]) } +func (c *sigctxt) fs() uint64 { return uint64(c.regs().gregs[_REG_FS]) } +func (c *sigctxt) gs() uint64 { return uint64(c.regs().gregs[_REG_GS]) } +func (c *sigctxt) sigcode() uint64 { return uint64(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return *(*uint64)(unsafe.Pointer(&c.info.__data[0])) } + +func (c *sigctxt) set_rip(x uint64) { c.regs().gregs[_REG_RIP] = int64(x) } +func (c *sigctxt) set_rsp(x uint64) { c.regs().gregs[_REG_RSP] = int64(x) } +func (c *sigctxt) set_sigcode(x uint64) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { + *(*uintptr)(unsafe.Pointer(&c.info.__data[0])) = uintptr(x) +} diff --git a/src/runtime/signal_solaris_amd64.h b/src/runtime/signal_solaris_amd64.h deleted file mode 100644 index c2e0a1549..000000000 --- a/src/runtime/signal_solaris_amd64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) - -#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RAX]) -#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RBX]) -#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RCX]) -#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RDX]) -#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RDI]) -#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RSI]) -#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RBP]) -#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RSP]) -#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R8]) -#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R9]) -#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R10]) -#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R11]) -#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R12]) -#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R13]) -#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R14]) -#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).gregs[REG_R15]) -#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RIP]) -#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).gregs[REG_RFLAGS]) - -#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).gregs[REG_CS]) -#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).gregs[REG_FS]) -#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gregs[REG_GS]) - -#define SIG_CODE0(info, ctxt) ((info)->si_code) -#define SIG_CODE1(info, ctxt) (*(uintptr*)&(info)->__data[0]) diff --git a/src/runtime/signal_unix.c b/src/runtime/signal_unix.c deleted file mode 100644 index 0e33ece49..000000000 --- a/src/runtime/signal_unix.c +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux netbsd openbsd solaris - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" - -extern SigTab runtime·sigtab[]; - -void -runtime·initsig(void) -{ - int32 i; - SigTab *t; - - // First call: basic setup. - for(i = 0; i<NSIG; i++) { - t = &runtime·sigtab[i]; - if((t->flags == 0) || (t->flags & SigDefault)) - continue; - - // For some signals, we respect an inherited SIG_IGN handler - // rather than insist on installing our own default handler. - // Even these signals can be fetched using the os/signal package. - switch(i) { - case SIGHUP: - case SIGINT: - if(runtime·getsig(i) == SIG_IGN) { - t->flags = SigNotify | SigIgnored; - continue; - } - } - - t->flags |= SigHandling; - runtime·setsig(i, runtime·sighandler, true); - } -} - -void -runtime·sigenable(uint32 sig) -{ - SigTab *t; - - if(sig >= NSIG) - return; - - t = &runtime·sigtab[sig]; - if((t->flags & SigNotify) && !(t->flags & SigHandling)) { - t->flags |= SigHandling; - if(runtime·getsig(sig) == SIG_IGN) - t->flags |= SigIgnored; - runtime·setsig(sig, runtime·sighandler, true); - } -} - -void -runtime·sigdisable(uint32 sig) -{ - SigTab *t; - - if(sig >= NSIG) - return; - - t = &runtime·sigtab[sig]; - if((t->flags & SigNotify) && (t->flags & SigHandling)) { - t->flags &= ~SigHandling; - if(t->flags & SigIgnored) - runtime·setsig(sig, SIG_IGN, true); - else - runtime·setsig(sig, SIG_DFL, true); - } -} - -void -runtime·resetcpuprofiler(int32 hz) -{ - Itimerval it; - - runtime·memclr((byte*)&it, sizeof it); - if(hz == 0) { - runtime·setitimer(ITIMER_PROF, &it, nil); - } else { - it.it_interval.tv_sec = 0; - it.it_interval.tv_usec = 1000000 / hz; - it.it_value = it.it_interval; - runtime·setitimer(ITIMER_PROF, &it, nil); - } - g->m->profilehz = hz; -} - -void -runtime·sigpipe(void) -{ - runtime·setsig(SIGPIPE, SIG_DFL, false); - runtime·raise(SIGPIPE); -} - -void -runtime·crash(void) -{ -#ifdef GOOS_darwin - // OS X core dumps are linear dumps of the mapped memory, - // from the first virtual byte to the last, with zeros in the gaps. - // Because of the way we arrange the address space on 64-bit systems, - // this means the OS X core file will be >128 GB and even on a zippy - // workstation can take OS X well over an hour to write (uninterruptible). - // Save users from making that mistake. - if(sizeof(void*) == 8) - return; -#endif - - runtime·unblocksignals(); - runtime·setsig(SIGABRT, SIG_DFL, false); - runtime·raise(SIGABRT); -} diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index ba77b6e7b..c457083dc 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -6,8 +6,6 @@ package runtime -func sigpipe() - func os_sigpipe() { - onM(sigpipe) + systemstack(sigpipe) } diff --git a/src/runtime/signals_darwin.h b/src/runtime/signals_darwin.h deleted file mode 100644 index 8761e1bd9..000000000 --- a/src/runtime/signals_darwin.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: terminal line hangup", - /* 2 */ N+K, "SIGINT: interrupt", - /* 3 */ N+T, "SIGQUIT: quit", - /* 4 */ T, "SIGILL: illegal instruction", - /* 5 */ T, "SIGTRAP: trace trap", - /* 6 */ N+T, "SIGABRT: abort", - /* 7 */ T, "SIGEMT: emulate instruction executed", - /* 8 */ P, "SIGFPE: floating-point exception", - /* 9 */ 0, "SIGKILL: kill", - /* 10 */ P, "SIGBUS: bus error", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ T, "SIGSYS: bad system call", - /* 13 */ N, "SIGPIPE: write to broken pipe", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: termination", - /* 16 */ N, "SIGURG: urgent condition on socket", - /* 17 */ 0, "SIGSTOP: stop", - /* 18 */ N+D, "SIGTSTP: keyboard stop", - /* 19 */ 0, "SIGCONT: continue after stop", - /* 20 */ N, "SIGCHLD: child status has changed", - /* 21 */ N+D, "SIGTTIN: background read from tty", - /* 22 */ N+D, "SIGTTOU: background write to tty", - /* 23 */ N, "SIGIO: i/o now possible", - /* 24 */ N, "SIGXCPU: cpu limit exceeded", - /* 25 */ N, "SIGXFSZ: file size limit exceeded", - /* 26 */ N, "SIGVTALRM: virtual alarm clock", - /* 27 */ N, "SIGPROF: profiling alarm clock", - /* 28 */ N, "SIGWINCH: window size change", - /* 29 */ N, "SIGINFO: status request from keyboard", - /* 30 */ N, "SIGUSR1: user-defined signal 1", - /* 31 */ N, "SIGUSR2: user-defined signal 2", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/signals_dragonfly.h b/src/runtime/signals_dragonfly.h deleted file mode 100644 index 07343a766..000000000 --- a/src/runtime/signals_dragonfly.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: terminal line hangup", - /* 2 */ N+K, "SIGINT: interrupt", - /* 3 */ N+T, "SIGQUIT: quit", - /* 4 */ T, "SIGILL: illegal instruction", - /* 5 */ T, "SIGTRAP: trace trap", - /* 6 */ N+T, "SIGABRT: abort", - /* 7 */ T, "SIGEMT: emulate instruction executed", - /* 8 */ P, "SIGFPE: floating-point exception", - /* 9 */ 0, "SIGKILL: kill", - /* 10 */ P, "SIGBUS: bus error", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ T, "SIGSYS: bad system call", - /* 13 */ N, "SIGPIPE: write to broken pipe", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: termination", - /* 16 */ N, "SIGURG: urgent condition on socket", - /* 17 */ 0, "SIGSTOP: stop", - /* 18 */ N+D, "SIGTSTP: keyboard stop", - /* 19 */ 0, "SIGCONT: continue after stop", - /* 20 */ N, "SIGCHLD: child status has changed", - /* 21 */ N+D, "SIGTTIN: background read from tty", - /* 22 */ N+D, "SIGTTOU: background write to tty", - /* 23 */ N, "SIGIO: i/o now possible", - /* 24 */ N, "SIGXCPU: cpu limit exceeded", - /* 25 */ N, "SIGXFSZ: file size limit exceeded", - /* 26 */ N, "SIGVTALRM: virtual alarm clock", - /* 27 */ N, "SIGPROF: profiling alarm clock", - /* 28 */ N, "SIGWINCH: window size change", - /* 29 */ N, "SIGINFO: status request from keyboard", - /* 30 */ N, "SIGUSR1: user-defined signal 1", - /* 31 */ N, "SIGUSR2: user-defined signal 2", - /* 32 */ N, "SIGTHR: reserved", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/signals_freebsd.h b/src/runtime/signals_freebsd.h deleted file mode 100644 index 39e0a947e..000000000 --- a/src/runtime/signals_freebsd.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: terminal line hangup", - /* 2 */ N+K, "SIGINT: interrupt", - /* 3 */ N+T, "SIGQUIT: quit", - /* 4 */ T, "SIGILL: illegal instruction", - /* 5 */ T, "SIGTRAP: trace trap", - /* 6 */ N+T, "SIGABRT: abort", - /* 7 */ T, "SIGEMT: emulate instruction executed", - /* 8 */ P, "SIGFPE: floating-point exception", - /* 9 */ 0, "SIGKILL: kill", - /* 10 */ P, "SIGBUS: bus error", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ N, "SIGSYS: bad system call", - /* 13 */ N, "SIGPIPE: write to broken pipe", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: termination", - /* 16 */ N, "SIGURG: urgent condition on socket", - /* 17 */ 0, "SIGSTOP: stop", - /* 18 */ N+D, "SIGTSTP: keyboard stop", - /* 19 */ 0, "SIGCONT: continue after stop", - /* 20 */ N, "SIGCHLD: child status has changed", - /* 21 */ N+D, "SIGTTIN: background read from tty", - /* 22 */ N+D, "SIGTTOU: background write to tty", - /* 23 */ N, "SIGIO: i/o now possible", - /* 24 */ N, "SIGXCPU: cpu limit exceeded", - /* 25 */ N, "SIGXFSZ: file size limit exceeded", - /* 26 */ N, "SIGVTALRM: virtual alarm clock", - /* 27 */ N, "SIGPROF: profiling alarm clock", - /* 28 */ N, "SIGWINCH: window size change", - /* 29 */ N, "SIGINFO: status request from keyboard", - /* 30 */ N, "SIGUSR1: user-defined signal 1", - /* 31 */ N, "SIGUSR2: user-defined signal 2", - /* 32 */ N, "SIGTHR: reserved", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/signals_linux.h b/src/runtime/signals_linux.h deleted file mode 100644 index 374107609..000000000 --- a/src/runtime/signals_linux.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: terminal line hangup", - /* 2 */ N+K, "SIGINT: interrupt", - /* 3 */ N+T, "SIGQUIT: quit", - /* 4 */ T, "SIGILL: illegal instruction", - /* 5 */ T, "SIGTRAP: trace trap", - /* 6 */ N+T, "SIGABRT: abort", - /* 7 */ P, "SIGBUS: bus error", - /* 8 */ P, "SIGFPE: floating-point exception", - /* 9 */ 0, "SIGKILL: kill", - /* 10 */ N, "SIGUSR1: user-defined signal 1", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ N, "SIGUSR2: user-defined signal 2", - /* 13 */ N, "SIGPIPE: write to broken pipe", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: termination", - /* 16 */ T, "SIGSTKFLT: stack fault", - /* 17 */ N, "SIGCHLD: child status has changed", - /* 18 */ 0, "SIGCONT: continue", - /* 19 */ 0, "SIGSTOP: stop, unblockable", - /* 20 */ N+D, "SIGTSTP: keyboard stop", - /* 21 */ N+D, "SIGTTIN: background read from tty", - /* 22 */ N+D, "SIGTTOU: background write to tty", - /* 23 */ N, "SIGURG: urgent condition on socket", - /* 24 */ N, "SIGXCPU: cpu limit exceeded", - /* 25 */ N, "SIGXFSZ: file size limit exceeded", - /* 26 */ N, "SIGVTALRM: virtual alarm clock", - /* 27 */ N, "SIGPROF: profiling alarm clock", - /* 28 */ N, "SIGWINCH: window size change", - /* 29 */ N, "SIGIO: i/o now possible", - /* 30 */ N, "SIGPWR: power failure restart", - /* 31 */ N, "SIGSYS: bad system call", - /* 32 */ 0, "signal 32", /* SIGCANCEL; see issue 6997 */ - /* 33 */ 0, "signal 33", /* SIGSETXID; see issue 3871 */ - /* 34 */ N, "signal 34", - /* 35 */ N, "signal 35", - /* 36 */ N, "signal 36", - /* 37 */ N, "signal 37", - /* 38 */ N, "signal 38", - /* 39 */ N, "signal 39", - /* 40 */ N, "signal 40", - /* 41 */ N, "signal 41", - /* 42 */ N, "signal 42", - /* 43 */ N, "signal 43", - /* 44 */ N, "signal 44", - /* 45 */ N, "signal 45", - /* 46 */ N, "signal 46", - /* 47 */ N, "signal 47", - /* 48 */ N, "signal 48", - /* 49 */ N, "signal 49", - /* 50 */ N, "signal 50", - /* 51 */ N, "signal 51", - /* 52 */ N, "signal 52", - /* 53 */ N, "signal 53", - /* 54 */ N, "signal 54", - /* 55 */ N, "signal 55", - /* 56 */ N, "signal 56", - /* 57 */ N, "signal 57", - /* 58 */ N, "signal 58", - /* 59 */ N, "signal 59", - /* 60 */ N, "signal 60", - /* 61 */ N, "signal 61", - /* 62 */ N, "signal 62", - /* 63 */ N, "signal 63", - /* 64 */ N, "signal 64", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/signals_openbsd.h b/src/runtime/signals_openbsd.h deleted file mode 100644 index 950a2fe62..000000000 --- a/src/runtime/signals_openbsd.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: terminal line hangup", - /* 2 */ N+K, "SIGINT: interrupt", - /* 3 */ N+T, "SIGQUIT: quit", - /* 4 */ T, "SIGILL: illegal instruction", - /* 5 */ T, "SIGTRAP: trace trap", - /* 6 */ N+T, "SIGABRT: abort", - /* 7 */ T, "SIGEMT: emulate instruction executed", - /* 8 */ P, "SIGFPE: floating-point exception", - /* 9 */ 0, "SIGKILL: kill", - /* 10 */ P, "SIGBUS: bus error", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ T, "SIGSYS: bad system call", - /* 13 */ N, "SIGPIPE: write to broken pipe", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: termination", - /* 16 */ N, "SIGURG: urgent condition on socket", - /* 17 */ 0, "SIGSTOP: stop", - /* 18 */ N+D, "SIGTSTP: keyboard stop", - /* 19 */ 0, "SIGCONT: continue after stop", - /* 20 */ N, "SIGCHLD: child status has changed", - /* 21 */ N+D, "SIGTTIN: background read from tty", - /* 22 */ N+D, "SIGTTOU: background write to tty", - /* 23 */ N, "SIGIO: i/o now possible", - /* 24 */ N, "SIGXCPU: cpu limit exceeded", - /* 25 */ N, "SIGXFSZ: file size limit exceeded", - /* 26 */ N, "SIGVTALRM: virtual alarm clock", - /* 27 */ N, "SIGPROF: profiling alarm clock", - /* 28 */ N, "SIGWINCH: window size change", - /* 29 */ N, "SIGINFO: status request from keyboard", - /* 30 */ N, "SIGUSR1: user-defined signal 1", - /* 31 */ N, "SIGUSR2: user-defined signal 2", - /* 32 */ N, "SIGTHR: reserved", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/signals_solaris.h b/src/runtime/signals_solaris.h deleted file mode 100644 index 1f0a65ea6..000000000 --- a/src/runtime/signals_solaris.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -#define N SigNotify -#define K SigKill -#define T SigThrow -#define P SigPanic -#define D SigDefault - -#pragma dataflag NOPTR -SigTab runtime·sigtab[] = { - /* 0 */ 0, "SIGNONE: no trap", - /* 1 */ N+K, "SIGHUP: hangup", - /* 2 */ N+K, "SIGINT: interrupt (rubout)", - /* 3 */ N+T, "SIGQUIT: quit (ASCII FS)", - /* 4 */ T, "SIGILL: illegal instruction (not reset when caught)", - /* 5 */ T, "SIGTRAP: trace trap (not reset when caught)", - /* 6 */ N+T, "SIGABRT: used by abort, replace SIGIOT in the future", - /* 7 */ T, "SIGEMT: EMT instruction", - /* 8 */ P, "SIGFPE: floating point exception", - /* 9 */ 0, "SIGKILL: kill (cannot be caught or ignored)", - /* 10 */ P, "SIGBUS: bus error", - /* 11 */ P, "SIGSEGV: segmentation violation", - /* 12 */ T, "SIGSYS: bad argument to system call", - /* 13 */ N, "SIGPIPE: write on a pipe with no one to read it", - /* 14 */ N, "SIGALRM: alarm clock", - /* 15 */ N+K, "SIGTERM: software termination signal from kill", - /* 16 */ N, "SIGUSR1: user defined signal 1", - /* 17 */ N, "SIGUSR2: user defined signal 2", - /* 18 */ N, "SIGCLD: child status change", - /* 18 */ N, "SIGCHLD: child status change alias (POSIX)", - /* 19 */ N, "SIGPWR: power-fail restart", - /* 20 */ N, "SIGWINCH: window size change", - /* 21 */ N, "SIGURG: urgent socket condition", - /* 22 */ N, "SIGPOLL: pollable event occured", - /* 23 */ N+D, "SIGSTOP: stop (cannot be caught or ignored)", - /* 24 */ 0, "SIGTSTP: user stop requested from tty", - /* 25 */ 0, "SIGCONT: stopped process has been continued", - /* 26 */ N+D, "SIGTTIN: background tty read attempted", - /* 27 */ N+D, "SIGTTOU: background tty write attempted", - /* 28 */ N, "SIGVTALRM: virtual timer expired", - /* 29 */ N, "SIGPROF: profiling timer expired", - /* 30 */ N, "SIGXCPU: exceeded cpu limit", - /* 31 */ N, "SIGXFSZ: exceeded file size limit", - /* 32 */ N, "SIGWAITING: reserved signal no longer used by", - /* 33 */ N, "SIGLWP: reserved signal no longer used by", - /* 34 */ N, "SIGFREEZE: special signal used by CPR", - /* 35 */ N, "SIGTHAW: special signal used by CPR", - /* 36 */ 0, "SIGCANCEL: reserved signal for thread cancellation", - /* 37 */ N, "SIGLOST: resource lost (eg, record-lock lost)", - /* 38 */ N, "SIGXRES: resource control exceeded", - /* 39 */ N, "SIGJVM1: reserved signal for Java Virtual Machine", - /* 40 */ N, "SIGJVM2: reserved signal for Java Virtual Machine", - - /* TODO(aram): what should be do about these signals? D or N? is this set static? */ - /* 41 */ N, "real time signal", - /* 42 */ N, "real time signal", - /* 43 */ N, "real time signal", - /* 44 */ N, "real time signal", - /* 45 */ N, "real time signal", - /* 46 */ N, "real time signal", - /* 47 */ N, "real time signal", - /* 48 */ N, "real time signal", - /* 49 */ N, "real time signal", - /* 50 */ N, "real time signal", - /* 51 */ N, "real time signal", - /* 52 */ N, "real time signal", - /* 53 */ N, "real time signal", - /* 54 */ N, "real time signal", - /* 55 */ N, "real time signal", - /* 56 */ N, "real time signal", - /* 57 */ N, "real time signal", - /* 58 */ N, "real time signal", - /* 59 */ N, "real time signal", - /* 60 */ N, "real time signal", - /* 61 */ N, "real time signal", - /* 62 */ N, "real time signal", - /* 63 */ N, "real time signal", - /* 64 */ N, "real time signal", - /* 65 */ N, "real time signal", - /* 66 */ N, "real time signal", - /* 67 */ N, "real time signal", - /* 68 */ N, "real time signal", - /* 69 */ N, "real time signal", - /* 70 */ N, "real time signal", - /* 71 */ N, "real time signal", - /* 72 */ N, "real time signal", -}; - -#undef N -#undef K -#undef T -#undef P -#undef D diff --git a/src/runtime/sigpanic_unix.go b/src/runtime/sigpanic_unix.go index 68079859b..7bf2c1540 100644 --- a/src/runtime/sigpanic_unix.go +++ b/src/runtime/sigpanic_unix.go @@ -6,8 +6,6 @@ package runtime -func signame(int32) *byte - func sigpanic() { g := getg() if !canpanic(g) { @@ -36,5 +34,10 @@ func sigpanic() { } panicfloat() } - panic(errorString(gostringnocopy(signame(g.sig)))) + + if g.sig >= uint32(len(sigtable)) { + // can't happen: we looked up g.sig in sigtable to decide to call sigpanic + gothrow("unexpected signal value") + } + panic(errorString(sigtable[g.sig].name)) } diff --git a/src/runtime/sigqueue.go b/src/runtime/sigqueue.go index 2d9c24d2d..82ead228f 100644 --- a/src/runtime/sigqueue.go +++ b/src/runtime/sigqueue.go @@ -45,7 +45,7 @@ const ( // Called from sighandler to send a signal back out of the signal handling thread. // Reports whether the signal was sent. If not, the caller typically crashes the program. -func sigsend(s int32) bool { +func sigsend(s uint32) bool { bit := uint32(1) << uint(s&31) if !sig.inuse || s < 0 || int(s) >= 32*len(sig.wanted) || sig.wanted[s/32]&bit == 0 { return false @@ -139,7 +139,7 @@ func signal_enable(s uint32) { return } sig.wanted[s/32] |= 1 << (s & 31) - sigenable_go(s) + sigenable(s) } // Must only be called from a single goroutine at a time. @@ -148,7 +148,7 @@ func signal_disable(s uint32) { return } sig.wanted[s/32] &^= 1 << (s & 31) - sigdisable_go(s) + sigdisable(s) } // This runs on a foreign stack, without an m or a g. No stack split. @@ -156,18 +156,3 @@ func signal_disable(s uint32) { func badsignal(sig uintptr) { cgocallback(unsafe.Pointer(funcPC(sigsend)), noescape(unsafe.Pointer(&sig)), unsafe.Sizeof(sig)) } - -func sigenable_m() -func sigdisable_m() - -func sigenable_go(s uint32) { - g := getg() - g.m.scalararg[0] = uintptr(s) - onM(sigenable_m) -} - -func sigdisable_go(s uint32) { - g := getg() - g.m.scalararg[0] = uintptr(s) - onM(sigdisable_m) -} diff --git a/src/runtime/slice.go b/src/runtime/slice.go index 171087d7f..93cea5cc3 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -22,11 +22,11 @@ func makeslice(t *slicetype, len64 int64, cap64 int64) sliceStruct { // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. len := int(len64) - if len64 < 0 || int64(len) != len64 || t.elem.size > 0 && uintptr(len) > maxmem/uintptr(t.elem.size) { + if len64 < 0 || int64(len) != len64 || t.elem.size > 0 && uintptr(len) > _MaxMem/uintptr(t.elem.size) { panic(errorString("makeslice: len out of range")) } cap := int(cap64) - if cap < len || int64(cap) != cap64 || t.elem.size > 0 && uintptr(cap) > maxmem/uintptr(t.elem.size) { + if cap < len || int64(cap) != cap64 || t.elem.size > 0 && uintptr(cap) > _MaxMem/uintptr(t.elem.size) { panic(errorString("makeslice: cap out of range")) } p := newarray(t.elem, uintptr(cap)) @@ -42,7 +42,7 @@ func growslice(t *slicetype, old sliceStruct, n int64) sliceStruct { cap64 := int64(old.cap) + n cap := int(cap64) - if int64(cap) != cap64 || cap < old.cap || t.elem.size > 0 && uintptr(cap) > maxmem/uintptr(t.elem.size) { + if int64(cap) != cap64 || cap < old.cap || t.elem.size > 0 && uintptr(cap) > _MaxMem/uintptr(t.elem.size) { panic(errorString("growslice: cap out of range")) } @@ -72,7 +72,7 @@ func growslice(t *slicetype, old sliceStruct, n int64) sliceStruct { } } - if uintptr(newcap) >= maxmem/uintptr(et.size) { + if uintptr(newcap) >= _MaxMem/uintptr(et.size) { panic(errorString("growslice: cap out of range")) } lenmem := uintptr(old.len) * uintptr(et.size) diff --git a/src/runtime/softfloat64.go b/src/runtime/softfloat64.go index 4fcf8f269..c157a14e2 100644 --- a/src/runtime/softfloat64.go +++ b/src/runtime/softfloat64.go @@ -340,7 +340,7 @@ func f32to64(f uint32) uint64 { return fpack64(fs64, uint64(fm)<<d, fe, 0) } -func fcmp64(f, g uint64) (cmp int, isnan bool) { +func fcmp64(f, g uint64) (cmp int32, isnan bool) { fs, fm, _, fi, fn := funpack64(f) gs, gm, _, gi, gn := funpack64(g) @@ -486,13 +486,13 @@ again2: // callable from C -func fadd64c(f, g uint64, ret *uint64) { *ret = fadd64(f, g) } -func fsub64c(f, g uint64, ret *uint64) { *ret = fsub64(f, g) } -func fmul64c(f, g uint64, ret *uint64) { *ret = fmul64(f, g) } -func fdiv64c(f, g uint64, ret *uint64) { *ret = fdiv64(f, g) } -func fneg64c(f uint64, ret *uint64) { *ret = fneg64(f) } -func f32to64c(f uint32, ret *uint64) { *ret = f32to64(f) } -func f64to32c(f uint64, ret *uint32) { *ret = f64to32(f) } -func fcmp64c(f, g uint64, ret *int, retnan *bool) { *ret, *retnan = fcmp64(f, g) } -func fintto64c(val int64, ret *uint64) { *ret = fintto64(val) } -func f64tointc(f uint64, ret *int64, retok *bool) { *ret, *retok = f64toint(f) } +func fadd64c(f, g uint64, ret *uint64) { *ret = fadd64(f, g) } +func fsub64c(f, g uint64, ret *uint64) { *ret = fsub64(f, g) } +func fmul64c(f, g uint64, ret *uint64) { *ret = fmul64(f, g) } +func fdiv64c(f, g uint64, ret *uint64) { *ret = fdiv64(f, g) } +func fneg64c(f uint64, ret *uint64) { *ret = fneg64(f) } +func f32to64c(f uint32, ret *uint64) { *ret = f32to64(f) } +func f64to32c(f uint64, ret *uint32) { *ret = f64to32(f) } +func fcmp64c(f, g uint64, ret *int32, retnan *bool) { *ret, *retnan = fcmp64(f, g) } +func fintto64c(val int64, ret *uint64) { *ret = fintto64(val) } +func f64tointc(f uint64, ret *int64, retok *bool) { *ret, *retok = f64toint(f) } diff --git a/src/runtime/softfloat64_test.go b/src/runtime/softfloat64_test.go index df63010fb..e10887283 100644 --- a/src/runtime/softfloat64_test.go +++ b/src/runtime/softfloat64_test.go @@ -182,7 +182,7 @@ func hwcmp(f, g float64) (cmp int, isnan bool) { func testcmp(t *testing.T, f, g float64) { hcmp, hisnan := hwcmp(f, g) scmp, sisnan := Fcmp64(math.Float64bits(f), math.Float64bits(g)) - if hcmp != scmp || hisnan != sisnan { + if int32(hcmp) != scmp || hisnan != sisnan { err(t, "cmp(%g, %g) = sw %v, %v, hw %v, %v\n", f, g, scmp, sisnan, hcmp, hisnan) } } diff --git a/src/runtime/softfloat_arm.c b/src/runtime/softfloat_arm.c deleted file mode 100644 index 3f3f33a19..000000000 --- a/src/runtime/softfloat_arm.c +++ /dev/null @@ -1,687 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Software floating point interpretaton of ARM 7500 FP instructions. -// The interpretation is not bit compatible with the 7500. -// It uses true little-endian doubles, while the 7500 used mixed-endian. - -#include "runtime.h" -#include "textflag.h" - -#define CPSR 14 -#define FLAGS_N (1U << 31) -#define FLAGS_Z (1U << 30) -#define FLAGS_C (1U << 29) -#define FLAGS_V (1U << 28) - -void runtime·abort(void); -void runtime·sqrtC(uint64, uint64*); - -static uint32 trace = 0; - -static void -fabort(void) -{ - if (1) { - runtime·printf("Unsupported floating point instruction\n"); - runtime·abort(); - } -} - -static void -putf(uint32 reg, uint32 val) -{ - g->m->freglo[reg] = val; -} - -static void -putd(uint32 reg, uint64 val) -{ - g->m->freglo[reg] = (uint32)val; - g->m->freghi[reg] = (uint32)(val>>32); -} - -static uint64 -getd(uint32 reg) -{ - return (uint64)g->m->freglo[reg] | ((uint64)g->m->freghi[reg]<<32); -} - -static void -fprint(void) -{ - uint32 i; - for (i = 0; i < 16; i++) { - runtime·printf("\tf%d:\t%X %X\n", i, g->m->freghi[i], g->m->freglo[i]); - } -} - -static uint32 -d2f(uint64 d) -{ - uint32 x; - - runtime·f64to32c(d, &x); - return x; -} - -static uint64 -f2d(uint32 f) -{ - uint64 x; - - runtime·f32to64c(f, &x); - return x; -} - -static uint32 -fstatus(bool nan, int32 cmp) -{ - if(nan) - return FLAGS_C | FLAGS_V; - if(cmp == 0) - return FLAGS_Z | FLAGS_C; - if(cmp < 0) - return FLAGS_N; - return FLAGS_C; -} - -// conditions array record the required CPSR cond field for the -// first 5 pairs of conditional execution opcodes -// higher 4 bits are must set, lower 4 bits are must clear -#pragma dataflag NOPTR -static const uint8 conditions[10/2] = { - [0/2] = (FLAGS_Z >> 24) | 0, // 0: EQ (Z set), 1: NE (Z clear) - [2/2] = (FLAGS_C >> 24) | 0, // 2: CS/HS (C set), 3: CC/LO (C clear) - [4/2] = (FLAGS_N >> 24) | 0, // 4: MI (N set), 5: PL (N clear) - [6/2] = (FLAGS_V >> 24) | 0, // 6: VS (V set), 7: VC (V clear) - [8/2] = (FLAGS_C >> 24) | - (FLAGS_Z >> 28), // 8: HI (C set and Z clear), 9: LS (C clear and Z set) -}; - -#define FAULT (0x80000000U) // impossible PC offset - -// returns number of words that the fp instruction -// is occupying, 0 if next instruction isn't float. -static uint32 -stepflt(uint32 *pc, uint32 *regs) -{ - uint32 i, opc, regd, regm, regn, cpsr; - int32 delta; - uint32 *addr; - uint64 uval; - int64 sval; - bool nan, ok; - int32 cmp; - M *m; - - // m is locked in vlop_arm.s, so g->m cannot change during this function call, - // so caching it in a local variable is safe. - m = g->m; - i = *pc; - - if(trace) - runtime·printf("stepflt %p %x (cpsr %x)\n", pc, i, regs[CPSR] >> 28); - - opc = i >> 28; - if(opc == 14) // common case first - goto execute; - cpsr = regs[CPSR] >> 28; - switch(opc) { - case 0: case 1: case 2: case 3: case 4: - case 5: case 6: case 7: case 8: case 9: - if(((cpsr & (conditions[opc/2] >> 4)) == (conditions[opc/2] >> 4)) && - ((cpsr & (conditions[opc/2] & 0xf)) == 0)) { - if(opc & 1) return 1; - } else { - if(!(opc & 1)) return 1; - } - break; - case 10: // GE (N == V) - case 11: // LT (N != V) - if((cpsr & (FLAGS_N >> 28)) == (cpsr & (FLAGS_V >> 28))) { - if(opc & 1) return 1; - } else { - if(!(opc & 1)) return 1; - } - break; - case 12: // GT (N == V and Z == 0) - case 13: // LE (N != V or Z == 1) - if((cpsr & (FLAGS_N >> 28)) == (cpsr & (FLAGS_V >> 28)) && - (cpsr & (FLAGS_Z >> 28)) == 0) { - if(opc & 1) return 1; - } else { - if(!(opc & 1)) return 1; - } - break; - case 14: // AL - break; - case 15: // shouldn't happen - return 0; - } - if(trace) - runtime·printf("conditional %x (cpsr %x) pass\n", opc, cpsr); - i = (0xeU << 28) | (i & 0xfffffff); - -execute: - // special cases - if((i&0xfffff000) == 0xe59fb000) { - // load r11 from pc-relative address. - // might be part of a floating point move - // (or might not, but no harm in simulating - // one instruction too many). - addr = (uint32*)((uint8*)pc + (i&0xfff) + 8); - regs[11] = addr[0]; - - if(trace) - runtime·printf("*** cpu R[%d] = *(%p) %x\n", - 11, addr, regs[11]); - return 1; - } - if(i == 0xe08bb00d) { - // add sp to r11. - // might be part of a large stack offset address - // (or might not, but again no harm done). - regs[11] += regs[13]; - - if(trace) - runtime·printf("*** cpu R[%d] += R[%d] %x\n", - 11, 13, regs[11]); - return 1; - } - if(i == 0xeef1fa10) { - regs[CPSR] = (regs[CPSR]&0x0fffffff) | m->fflag; - - if(trace) - runtime·printf("*** fpsr R[CPSR] = F[CPSR] %x\n", regs[CPSR]); - return 1; - } - if((i&0xff000000) == 0xea000000) { - // unconditional branch - // can happen in the middle of floating point - // if the linker decides it is time to lay down - // a sequence of instruction stream constants. - delta = i&0xffffff; - delta = (delta<<8) >> 8; // sign extend - - if(trace) - runtime·printf("*** cpu PC += %x\n", (delta+2)*4); - return delta+2; - } - - goto stage1; - -stage1: // load/store regn is cpureg, regm is 8bit offset - regd = i>>12 & 0xf; - regn = i>>16 & 0xf; - regm = (i & 0xff) << 2; // PLUS or MINUS ?? - - switch(i & 0xfff00f00) { - default: - goto stage2; - - case 0xed900a00: // single load - addr = (uint32*)(regs[regn] + regm); - if((uintptr)addr < 4096) { - if(trace) - runtime·printf("*** load @%p => fault\n", addr); - return FAULT; - } - m->freglo[regd] = addr[0]; - - if(trace) - runtime·printf("*** load F[%d] = %x\n", - regd, m->freglo[regd]); - break; - - case 0xed900b00: // double load - addr = (uint32*)(regs[regn] + regm); - if((uintptr)addr < 4096) { - if(trace) - runtime·printf("*** double load @%p => fault\n", addr); - return FAULT; - } - m->freglo[regd] = addr[0]; - m->freghi[regd] = addr[1]; - - if(trace) - runtime·printf("*** load D[%d] = %x-%x\n", - regd, m->freghi[regd], m->freglo[regd]); - break; - - case 0xed800a00: // single store - addr = (uint32*)(regs[regn] + regm); - if((uintptr)addr < 4096) { - if(trace) - runtime·printf("*** store @%p => fault\n", addr); - return FAULT; - } - addr[0] = m->freglo[regd]; - - if(trace) - runtime·printf("*** *(%p) = %x\n", - addr, addr[0]); - break; - - case 0xed800b00: // double store - addr = (uint32*)(regs[regn] + regm); - if((uintptr)addr < 4096) { - if(trace) - runtime·printf("*** double store @%p => fault\n", addr); - return FAULT; - } - addr[0] = m->freglo[regd]; - addr[1] = m->freghi[regd]; - - if(trace) - runtime·printf("*** *(%p) = %x-%x\n", - addr, addr[1], addr[0]); - break; - } - return 1; - -stage2: // regd, regm, regn are 4bit variables - regm = i>>0 & 0xf; - switch(i & 0xfff00ff0) { - default: - goto stage3; - - case 0xf3000110: // veor - m->freglo[regd] = m->freglo[regm]^m->freglo[regn]; - m->freghi[regd] = m->freghi[regm]^m->freghi[regn]; - - if(trace) - runtime·printf("*** veor D[%d] = %x-%x\n", - regd, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb00b00: // D[regd] = const(regn,regm) - regn = (regn<<4) | regm; - regm = 0x40000000UL; - if(regn & 0x80) - regm |= 0x80000000UL; - if(regn & 0x40) - regm ^= 0x7fc00000UL; - regm |= (regn & 0x3f) << 16; - m->freglo[regd] = 0; - m->freghi[regd] = regm; - - if(trace) - runtime·printf("*** immed D[%d] = %x-%x\n", - regd, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb00a00: // F[regd] = const(regn,regm) - regn = (regn<<4) | regm; - regm = 0x40000000UL; - if(regn & 0x80) - regm |= 0x80000000UL; - if(regn & 0x40) - regm ^= 0x7e000000UL; - regm |= (regn & 0x3f) << 19; - m->freglo[regd] = regm; - - if(trace) - runtime·printf("*** immed D[%d] = %x\n", - regd, m->freglo[regd]); - break; - - case 0xee300b00: // D[regd] = D[regn]+D[regm] - runtime·fadd64c(getd(regn), getd(regm), &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** add D[%d] = D[%d]+D[%d] %x-%x\n", - regd, regn, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xee300a00: // F[regd] = F[regn]+F[regm] - runtime·fadd64c(f2d(m->freglo[regn]), f2d(m->freglo[regm]), &uval); - m->freglo[regd] = d2f(uval); - - if(trace) - runtime·printf("*** add F[%d] = F[%d]+F[%d] %x\n", - regd, regn, regm, m->freglo[regd]); - break; - - case 0xee300b40: // D[regd] = D[regn]-D[regm] - runtime·fsub64c(getd(regn), getd(regm), &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** sub D[%d] = D[%d]-D[%d] %x-%x\n", - regd, regn, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xee300a40: // F[regd] = F[regn]-F[regm] - runtime·fsub64c(f2d(m->freglo[regn]), f2d(m->freglo[regm]), &uval); - m->freglo[regd] = d2f(uval); - - if(trace) - runtime·printf("*** sub F[%d] = F[%d]-F[%d] %x\n", - regd, regn, regm, m->freglo[regd]); - break; - - case 0xee200b00: // D[regd] = D[regn]*D[regm] - runtime·fmul64c(getd(regn), getd(regm), &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** mul D[%d] = D[%d]*D[%d] %x-%x\n", - regd, regn, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xee200a00: // F[regd] = F[regn]*F[regm] - runtime·fmul64c(f2d(m->freglo[regn]), f2d(m->freglo[regm]), &uval); - m->freglo[regd] = d2f(uval); - - if(trace) - runtime·printf("*** mul F[%d] = F[%d]*F[%d] %x\n", - regd, regn, regm, m->freglo[regd]); - break; - - case 0xee800b00: // D[regd] = D[regn]/D[regm] - runtime·fdiv64c(getd(regn), getd(regm), &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** div D[%d] = D[%d]/D[%d] %x-%x\n", - regd, regn, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xee800a00: // F[regd] = F[regn]/F[regm] - runtime·fdiv64c(f2d(m->freglo[regn]), f2d(m->freglo[regm]), &uval); - m->freglo[regd] = d2f(uval); - - if(trace) - runtime·printf("*** div F[%d] = F[%d]/F[%d] %x\n", - regd, regn, regm, m->freglo[regd]); - break; - - case 0xee000b10: // S[regn] = R[regd] (MOVW) (regm ignored) - m->freglo[regn] = regs[regd]; - - if(trace) - runtime·printf("*** cpy S[%d] = R[%d] %x\n", - regn, regd, m->freglo[regn]); - break; - - case 0xee100b10: // R[regd] = S[regn] (MOVW) (regm ignored) - regs[regd] = m->freglo[regn]; - - if(trace) - runtime·printf("*** cpy R[%d] = S[%d] %x\n", - regd, regn, regs[regd]); - break; - } - return 1; - -stage3: // regd, regm are 4bit variables - switch(i & 0xffff0ff0) { - default: - goto done; - - case 0xeeb00a40: // F[regd] = F[regm] (MOVF) - m->freglo[regd] = m->freglo[regm]; - - if(trace) - runtime·printf("*** F[%d] = F[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeeb00b40: // D[regd] = D[regm] (MOVD) - m->freglo[regd] = m->freglo[regm]; - m->freghi[regd] = m->freghi[regm]; - - if(trace) - runtime·printf("*** D[%d] = D[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb10bc0: // D[regd] = sqrt D[regm] - runtime·sqrtC(getd(regm), &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** D[%d] = sqrt D[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb00bc0: // D[regd] = abs D[regm] - m->freglo[regd] = m->freglo[regm]; - m->freghi[regd] = m->freghi[regm] & ((1<<31)-1); - - if(trace) - runtime·printf("*** D[%d] = abs D[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb00ac0: // F[regd] = abs F[regm] - m->freglo[regd] = m->freglo[regm] & ((1<<31)-1); - - if(trace) - runtime·printf("*** F[%d] = abs F[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeeb40bc0: // D[regd] :: D[regm] (CMPD) - runtime·fcmp64c(getd(regd), getd(regm), &cmp, &nan); - m->fflag = fstatus(nan, cmp); - - if(trace) - runtime·printf("*** cmp D[%d]::D[%d] %x\n", - regd, regm, m->fflag); - break; - - case 0xeeb40ac0: // F[regd] :: F[regm] (CMPF) - runtime·fcmp64c(f2d(m->freglo[regd]), f2d(m->freglo[regm]), &cmp, &nan); - m->fflag = fstatus(nan, cmp); - - if(trace) - runtime·printf("*** cmp F[%d]::F[%d] %x\n", - regd, regm, m->fflag); - break; - - case 0xeeb70ac0: // D[regd] = F[regm] (MOVFD) - putd(regd, f2d(m->freglo[regm])); - - if(trace) - runtime·printf("*** f2d D[%d]=F[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb70bc0: // F[regd] = D[regm] (MOVDF) - m->freglo[regd] = d2f(getd(regm)); - - if(trace) - runtime·printf("*** d2f F[%d]=D[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeebd0ac0: // S[regd] = F[regm] (MOVFW) - runtime·f64tointc(f2d(m->freglo[regm]), &sval, &ok); - if(!ok || (int32)sval != sval) - sval = 0; - m->freglo[regd] = sval; - - if(trace) - runtime·printf("*** fix S[%d]=F[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeebc0ac0: // S[regd] = F[regm] (MOVFW.U) - runtime·f64tointc(f2d(m->freglo[regm]), &sval, &ok); - if(!ok || (uint32)sval != sval) - sval = 0; - m->freglo[regd] = sval; - - if(trace) - runtime·printf("*** fix unsigned S[%d]=F[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeebd0bc0: // S[regd] = D[regm] (MOVDW) - runtime·f64tointc(getd(regm), &sval, &ok); - if(!ok || (int32)sval != sval) - sval = 0; - m->freglo[regd] = sval; - - if(trace) - runtime·printf("*** fix S[%d]=D[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeebc0bc0: // S[regd] = D[regm] (MOVDW.U) - runtime·f64tointc(getd(regm), &sval, &ok); - if(!ok || (uint32)sval != sval) - sval = 0; - m->freglo[regd] = sval; - - if(trace) - runtime·printf("*** fix unsigned S[%d]=D[%d] %x\n", - regd, regm, m->freglo[regd]); - break; - - case 0xeeb80ac0: // D[regd] = S[regm] (MOVWF) - cmp = m->freglo[regm]; - if(cmp < 0) { - runtime·fintto64c(-cmp, &uval); - putf(regd, d2f(uval)); - m->freglo[regd] ^= 0x80000000; - } else { - runtime·fintto64c(cmp, &uval); - putf(regd, d2f(uval)); - } - - if(trace) - runtime·printf("*** float D[%d]=S[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb80a40: // D[regd] = S[regm] (MOVWF.U) - runtime·fintto64c(m->freglo[regm], &uval); - putf(regd, d2f(uval)); - - if(trace) - runtime·printf("*** float unsigned D[%d]=S[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb80bc0: // D[regd] = S[regm] (MOVWD) - cmp = m->freglo[regm]; - if(cmp < 0) { - runtime·fintto64c(-cmp, &uval); - putd(regd, uval); - m->freghi[regd] ^= 0x80000000; - } else { - runtime·fintto64c(cmp, &uval); - putd(regd, uval); - } - - if(trace) - runtime·printf("*** float D[%d]=S[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - - case 0xeeb80b40: // D[regd] = S[regm] (MOVWD.U) - runtime·fintto64c(m->freglo[regm], &uval); - putd(regd, uval); - - if(trace) - runtime·printf("*** float unsigned D[%d]=S[%d] %x-%x\n", - regd, regm, m->freghi[regd], m->freglo[regd]); - break; - } - return 1; - -done: - if((i&0xff000000) == 0xee000000 || - (i&0xff000000) == 0xed000000) { - runtime·printf("stepflt %p %x\n", pc, i); - fabort(); - } - return 0; -} - -typedef struct Sfregs Sfregs; - -// NOTE: These are all recorded as pointers because they are possibly live registers, -// and we don't know what they contain. Recording them as pointers should be -// safer than not. -struct Sfregs -{ - uint32 *r0; - uint32 *r1; - uint32 *r2; - uint32 *r3; - uint32 *r4; - uint32 *r5; - uint32 *r6; - uint32 *r7; - uint32 *r8; - uint32 *r9; - uint32 *r10; - uint32 *r11; - uint32 *r12; - uint32 *r13; - uint32 cspr; -}; - -static void sfloat2(void); -void _sfloatpanic(void); - -#pragma textflag NOSPLIT -uint32* -runtime·_sfloat2(uint32 *pc, Sfregs regs) -{ - void (*fn)(void); - - g->m->ptrarg[0] = pc; - g->m->ptrarg[1] = ®s; - fn = sfloat2; - runtime·onM(&fn); - pc = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - return pc; -} - -static void -sfloat2(void) -{ - uint32 *pc; - G *curg; - Sfregs *regs; - int32 skip; - bool first; - - pc = g->m->ptrarg[0]; - regs = g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - - first = true; - while(skip = stepflt(pc, (uint32*)®s->r0)) { - first = false; - if(skip == FAULT) { - // Encountered bad address in store/load. - // Record signal information and return to assembly - // trampoline that fakes the call. - enum { SIGSEGV = 11 }; - curg = g->m->curg; - curg->sig = SIGSEGV; - curg->sigcode0 = 0; - curg->sigcode1 = 0; - curg->sigpc = (uint32)pc; - pc = (uint32*)_sfloatpanic; - break; - } - pc += skip; - } - if(first) { - runtime·printf("sfloat2 %p %x\n", pc, *pc); - fabort(); // not ok to fail first instruction - } - - g->m->ptrarg[0] = pc; -} diff --git a/src/runtime/softfloat_arm.go b/src/runtime/softfloat_arm.go new file mode 100644 index 000000000..efee31c4c --- /dev/null +++ b/src/runtime/softfloat_arm.go @@ -0,0 +1,645 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Software floating point interpretaton of ARM 7500 FP instructions. +// The interpretation is not bit compatible with the 7500. +// It uses true little-endian doubles, while the 7500 used mixed-endian. + +package runtime + +import "unsafe" + +const ( + _CPSR = 14 + _FLAGS_N = 1 << 31 + _FLAGS_Z = 1 << 30 + _FLAGS_C = 1 << 29 + _FLAGS_V = 1 << 28 +) + +var fptrace = 0 + +func fabort() { + gothrow("unsupported floating point instruction") +} + +func fputf(reg uint32, val uint32) { + _g_ := getg() + _g_.m.freglo[reg] = val +} + +func fputd(reg uint32, val uint64) { + _g_ := getg() + _g_.m.freglo[reg] = uint32(val) + _g_.m.freghi[reg] = uint32(val >> 32) +} + +func fgetd(reg uint32) uint64 { + _g_ := getg() + return uint64(_g_.m.freglo[reg]) | uint64(_g_.m.freghi[reg])<<32 +} + +func fprintregs() { + _g_ := getg() + for i := range _g_.m.freglo { + print("\tf", i, ":\t", hex(_g_.m.freghi[i]), " ", hex(_g_.m.freglo[i]), "\n") + } +} + +func fstatus(nan bool, cmp int32) uint32 { + if nan { + return _FLAGS_C | _FLAGS_V + } + if cmp == 0 { + return _FLAGS_Z | _FLAGS_C + } + if cmp < 0 { + return _FLAGS_N + } + return _FLAGS_C +} + +// conditions array record the required CPSR cond field for the +// first 5 pairs of conditional execution opcodes +// higher 4 bits are must set, lower 4 bits are must clear +var conditions = [10 / 2]uint32{ + 0 / 2: _FLAGS_Z>>24 | 0, // 0: EQ (Z set), 1: NE (Z clear) + 2 / 2: _FLAGS_C>>24 | 0, // 2: CS/HS (C set), 3: CC/LO (C clear) + 4 / 2: _FLAGS_N>>24 | 0, // 4: MI (N set), 5: PL (N clear) + 6 / 2: _FLAGS_V>>24 | 0, // 6: VS (V set), 7: VC (V clear) + 8 / 2: _FLAGS_C>>24 | + _FLAGS_Z>>28, +} + +const _FAULT = 0x80000000 // impossible PC offset + +// returns number of words that the fp instruction +// is occupying, 0 if next instruction isn't float. +func stepflt(pc *uint32, regs *[15]uint32) uint32 { + var ( + i, opc, regd, regm, regn, cpsr uint32 + cmp, delta int32 + uval uint64 + sval int64 + nan, ok bool + ) + + // m is locked in vlop_arm.s, so g.m cannot change during this function call, + // so caching it in a local variable is safe. + m := getg().m + i = *pc + + if fptrace > 0 { + print("stepflt ", pc, " ", hex(i), " (cpsr ", hex(regs[_CPSR]>>28), ")\n") + } + + opc = i >> 28 + if opc == 14 { // common case first + goto execute + } + + cpsr = regs[_CPSR] >> 28 + switch opc { + case 0, 1, 2, 3, 4, 5, 6, 7, 8, 9: + if cpsr&(conditions[opc/2]>>4) == conditions[opc/2]>>4 && + cpsr&(conditions[opc/2]&0xf) == 0 { + if opc&1 != 0 { + return 1 + } + } else { + if opc&1 == 0 { + return 1 + } + } + + case 10, 11: // GE (N == V), LT (N != V) + if cpsr&(_FLAGS_N>>28) == cpsr&(_FLAGS_V>>28) { + if opc&1 != 0 { + return 1 + } + } else { + if opc&1 == 0 { + return 1 + } + } + + case 12, 13: // GT (N == V and Z == 0), LE (N != V or Z == 1) + if cpsr&(_FLAGS_N>>28) == cpsr&(_FLAGS_V>>28) && + cpsr&(_FLAGS_Z>>28) == 0 { + if opc&1 != 0 { + return 1 + } + } else { + if opc&1 == 0 { + return 1 + } + } + + case 14: // AL + // ok + + case 15: // shouldn't happen + return 0 + } + + if fptrace > 0 { + print("conditional ", hex(opc), " (cpsr ", hex(cpsr), ") pass\n") + } + i = 0xe<<28 | i&(1<<28-1) + +execute: + // special cases + if i&0xfffff000 == 0xe59fb000 { + // load r11 from pc-relative address. + // might be part of a floating point move + // (or might not, but no harm in simulating + // one instruction too many). + addr := (*[1]uint32)(add(unsafe.Pointer(pc), uintptr(i&0xfff+8))) + regs[11] = addr[0] + + if fptrace > 0 { + print("*** cpu R[11] = *(", addr, ") ", hex(regs[11]), "\n") + } + return 1 + } + if i == 0xe08bb00d { + // add sp to r11. + // might be part of a large stack offset address + // (or might not, but again no harm done). + regs[11] += regs[13] + + if fptrace > 0 { + print("*** cpu R[11] += R[13] ", hex(regs[11]), "\n") + } + return 1 + } + if i == 0xeef1fa10 { + regs[_CPSR] = regs[_CPSR]&0x0fffffff | m.fflag + + if fptrace > 0 { + print("*** fpsr R[CPSR] = F[CPSR] ", hex(regs[_CPSR]), "\n") + } + return 1 + } + if i&0xff000000 == 0xea000000 { + // unconditional branch + // can happen in the middle of floating point + // if the linker decides it is time to lay down + // a sequence of instruction stream constants. + delta = int32(i&0xffffff) << 8 >> 8 // sign extend + + if fptrace > 0 { + print("*** cpu PC += ", hex((delta+2)*4), "\n") + } + return uint32(delta + 2) + } + + goto stage1 + +stage1: // load/store regn is cpureg, regm is 8bit offset + regd = i >> 12 & 0xf + regn = i >> 16 & 0xf + regm = i & 0xff << 2 // PLUS or MINUS ?? + + switch i & 0xfff00f00 { + default: + goto stage2 + + case 0xed900a00: // single load + uaddr := uintptr(regs[regn] + regm) + if uaddr < 4096 { + if fptrace > 0 { + print("*** load @", hex(uaddr), " => fault\n") + } + return _FAULT + } + addr := (*[1]uint32)(unsafe.Pointer(uaddr)) + m.freglo[regd] = addr[0] + + if fptrace > 0 { + print("*** load F[", regd, "] = ", hex(m.freglo[regd]), "\n") + } + break + + case 0xed900b00: // double load + uaddr := uintptr(regs[regn] + regm) + if uaddr < 4096 { + if fptrace > 0 { + print("*** double load @", hex(uaddr), " => fault\n") + } + return _FAULT + } + addr := (*[2]uint32)(unsafe.Pointer(uaddr)) + m.freglo[regd] = addr[0] + m.freghi[regd] = addr[1] + + if fptrace > 0 { + print("*** load D[", regd, "] = ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xed800a00: // single store + uaddr := uintptr(regs[regn] + regm) + if uaddr < 4096 { + if fptrace > 0 { + print("*** store @", hex(uaddr), " => fault\n") + } + return _FAULT + } + addr := (*[1]uint32)(unsafe.Pointer(uaddr)) + addr[0] = m.freglo[regd] + + if fptrace > 0 { + print("*** *(", addr, ") = ", hex(addr[0]), "\n") + } + break + + case 0xed800b00: // double store + uaddr := uintptr(regs[regn] + regm) + if uaddr < 4096 { + if fptrace > 0 { + print("*** double store @", hex(uaddr), " => fault\n") + } + return _FAULT + } + addr := (*[2]uint32)(unsafe.Pointer(uaddr)) + addr[0] = m.freglo[regd] + addr[1] = m.freghi[regd] + + if fptrace > 0 { + print("*** *(", addr, ") = ", hex(addr[1]), "-", hex(addr[0]), "\n") + } + break + } + return 1 + +stage2: // regd, regm, regn are 4bit variables + regm = i >> 0 & 0xf + switch i & 0xfff00ff0 { + default: + goto stage3 + + case 0xf3000110: // veor + m.freglo[regd] = m.freglo[regm] ^ m.freglo[regn] + m.freghi[regd] = m.freghi[regm] ^ m.freghi[regn] + + if fptrace > 0 { + print("*** veor D[", regd, "] = ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb00b00: // D[regd] = const(regn,regm) + regn = regn<<4 | regm + regm = 0x40000000 + if regn&0x80 != 0 { + regm |= 0x80000000 + } + if regn&0x40 != 0 { + regm ^= 0x7fc00000 + } + regm |= regn & 0x3f << 16 + m.freglo[regd] = 0 + m.freghi[regd] = regm + + if fptrace > 0 { + print("*** immed D[", regd, "] = ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb00a00: // F[regd] = const(regn,regm) + regn = regn<<4 | regm + regm = 0x40000000 + if regn&0x80 != 0 { + regm |= 0x80000000 + } + if regn&0x40 != 0 { + regm ^= 0x7e000000 + } + regm |= regn & 0x3f << 19 + m.freglo[regd] = regm + + if fptrace > 0 { + print("*** immed D[", regd, "] = ", hex(m.freglo[regd]), "\n") + } + break + + case 0xee300b00: // D[regd] = D[regn]+D[regm] + fadd64c(fgetd(regn), fgetd(regm), &uval) + fputd(regd, uval) + + if fptrace > 0 { + print("*** add D[", regd, "] = D[", regn, "]+D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xee300a00: // F[regd] = F[regn]+F[regm] + fadd64c(f32to64(m.freglo[regn]), f32to64(m.freglo[regm]), &uval) + m.freglo[regd] = f64to32(uval) + + if fptrace > 0 { + print("*** add F[", regd, "] = F[", regn, "]+F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xee300b40: // D[regd] = D[regn]-D[regm] + fsub64c(fgetd(regn), fgetd(regm), &uval) + fputd(regd, uval) + + if fptrace > 0 { + print("*** sub D[", regd, "] = D[", regn, "]-D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xee300a40: // F[regd] = F[regn]-F[regm] + fsub64c(f32to64(m.freglo[regn]), f32to64(m.freglo[regm]), &uval) + m.freglo[regd] = f64to32(uval) + + if fptrace > 0 { + print("*** sub F[", regd, "] = F[", regn, "]-F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xee200b00: // D[regd] = D[regn]*D[regm] + fmul64c(fgetd(regn), fgetd(regm), &uval) + fputd(regd, uval) + + if fptrace > 0 { + print("*** mul D[", regd, "] = D[", regn, "]*D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xee200a00: // F[regd] = F[regn]*F[regm] + fmul64c(f32to64(m.freglo[regn]), f32to64(m.freglo[regm]), &uval) + m.freglo[regd] = f64to32(uval) + + if fptrace > 0 { + print("*** mul F[", regd, "] = F[", regn, "]*F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xee800b00: // D[regd] = D[regn]/D[regm] + fdiv64c(fgetd(regn), fgetd(regm), &uval) + fputd(regd, uval) + + if fptrace > 0 { + print("*** div D[", regd, "] = D[", regn, "]/D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xee800a00: // F[regd] = F[regn]/F[regm] + fdiv64c(f32to64(m.freglo[regn]), f32to64(m.freglo[regm]), &uval) + m.freglo[regd] = f64to32(uval) + + if fptrace > 0 { + print("*** div F[", regd, "] = F[", regn, "]/F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xee000b10: // S[regn] = R[regd] (MOVW) (regm ignored) + m.freglo[regn] = regs[regd] + + if fptrace > 0 { + print("*** cpy S[", regn, "] = R[", regd, "] ", hex(m.freglo[regn]), "\n") + } + break + + case 0xee100b10: // R[regd] = S[regn] (MOVW) (regm ignored) + regs[regd] = m.freglo[regn] + + if fptrace > 0 { + print("*** cpy R[", regd, "] = S[", regn, "] ", hex(regs[regd]), "\n") + } + break + } + return 1 + +stage3: // regd, regm are 4bit variables + switch i & 0xffff0ff0 { + default: + goto done + + case 0xeeb00a40: // F[regd] = F[regm] (MOVF) + m.freglo[regd] = m.freglo[regm] + + if fptrace > 0 { + print("*** F[", regd, "] = F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb00b40: // D[regd] = D[regm] (MOVD) + m.freglo[regd] = m.freglo[regm] + m.freghi[regd] = m.freghi[regm] + + if fptrace > 0 { + print("*** D[", regd, "] = D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb10bc0: // D[regd] = sqrt D[regm] + uval = float64bits(sqrt(float64frombits(fgetd(regm)))) + fputd(regd, uval) + + if fptrace > 0 { + print("*** D[", regd, "] = sqrt D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb00bc0: // D[regd] = abs D[regm] + m.freglo[regd] = m.freglo[regm] + m.freghi[regd] = m.freghi[regm] & (1<<31 - 1) + + if fptrace > 0 { + print("*** D[", regd, "] = abs D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb00ac0: // F[regd] = abs F[regm] + m.freglo[regd] = m.freglo[regm] & (1<<31 - 1) + + if fptrace > 0 { + print("*** F[", regd, "] = abs F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb40bc0: // D[regd] :: D[regm] (CMPD) + fcmp64c(fgetd(regd), fgetd(regm), &cmp, &nan) + m.fflag = fstatus(nan, cmp) + + if fptrace > 0 { + print("*** cmp D[", regd, "]::D[", regm, "] ", hex(m.fflag), "\n") + } + break + + case 0xeeb40ac0: // F[regd] :: F[regm] (CMPF) + fcmp64c(f32to64(m.freglo[regd]), f32to64(m.freglo[regm]), &cmp, &nan) + m.fflag = fstatus(nan, cmp) + + if fptrace > 0 { + print("*** cmp F[", regd, "]::F[", regm, "] ", hex(m.fflag), "\n") + } + break + + case 0xeeb70ac0: // D[regd] = F[regm] (MOVFD) + fputd(regd, f32to64(m.freglo[regm])) + + if fptrace > 0 { + print("*** f2d D[", regd, "]=F[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb70bc0: // F[regd] = D[regm] (MOVDF) + m.freglo[regd] = f64to32(fgetd(regm)) + + if fptrace > 0 { + print("*** d2f F[", regd, "]=D[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeebd0ac0: // S[regd] = F[regm] (MOVFW) + f64tointc(f32to64(m.freglo[regm]), &sval, &ok) + if !ok || int64(int32(sval)) != sval { + sval = 0 + } + m.freglo[regd] = uint32(sval) + if fptrace > 0 { + print("*** fix S[", regd, "]=F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeebc0ac0: // S[regd] = F[regm] (MOVFW.U) + f64tointc(f32to64(m.freglo[regm]), &sval, &ok) + if !ok || int64(uint32(sval)) != sval { + sval = 0 + } + m.freglo[regd] = uint32(sval) + + if fptrace > 0 { + print("*** fix unsigned S[", regd, "]=F[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeebd0bc0: // S[regd] = D[regm] (MOVDW) + f64tointc(fgetd(regm), &sval, &ok) + if !ok || int64(int32(sval)) != sval { + sval = 0 + } + m.freglo[regd] = uint32(sval) + + if fptrace > 0 { + print("*** fix S[", regd, "]=D[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeebc0bc0: // S[regd] = D[regm] (MOVDW.U) + f64tointc(fgetd(regm), &sval, &ok) + if !ok || int64(uint32(sval)) != sval { + sval = 0 + } + m.freglo[regd] = uint32(sval) + + if fptrace > 0 { + print("*** fix unsigned S[", regd, "]=D[", regm, "] ", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb80ac0: // D[regd] = S[regm] (MOVWF) + cmp = int32(m.freglo[regm]) + if cmp < 0 { + fintto64c(int64(-cmp), &uval) + fputf(regd, f64to32(uval)) + m.freglo[regd] ^= 0x80000000 + } else { + fintto64c(int64(cmp), &uval) + fputf(regd, f64to32(uval)) + } + + if fptrace > 0 { + print("*** float D[", regd, "]=S[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb80a40: // D[regd] = S[regm] (MOVWF.U) + fintto64c(int64(m.freglo[regm]), &uval) + fputf(regd, f64to32(uval)) + + if fptrace > 0 { + print("*** float unsigned D[", regd, "]=S[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb80bc0: // D[regd] = S[regm] (MOVWD) + cmp = int32(m.freglo[regm]) + if cmp < 0 { + fintto64c(int64(-cmp), &uval) + fputd(regd, uval) + m.freghi[regd] ^= 0x80000000 + } else { + fintto64c(int64(cmp), &uval) + fputd(regd, uval) + } + + if fptrace > 0 { + print("*** float D[", regd, "]=S[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + + case 0xeeb80b40: // D[regd] = S[regm] (MOVWD.U) + fintto64c(int64(m.freglo[regm]), &uval) + fputd(regd, uval) + + if fptrace > 0 { + print("*** float unsigned D[", regd, "]=S[", regm, "] ", hex(m.freghi[regd]), "-", hex(m.freglo[regd]), "\n") + } + break + } + return 1 + +done: + if i&0xff000000 == 0xee000000 || + i&0xff000000 == 0xed000000 { + print("stepflt ", pc, " ", hex(i), "\n") + fabort() + } + return 0 +} + +//go:nosplit +func _sfloat2(pc uint32, regs [15]uint32) (newpc uint32) { + systemstack(func() { + newpc = sfloat2(pc, ®s) + }) + return +} + +func _sfloatpanic() + +func sfloat2(pc uint32, regs *[15]uint32) uint32 { + first := true + for { + skip := stepflt((*uint32)(unsafe.Pointer(uintptr(pc))), regs) + if skip == 0 { + break + } + first = false + if skip == _FAULT { + // Encountered bad address in store/load. + // Record signal information and return to assembly + // trampoline that fakes the call. + const SIGSEGV = 11 + curg := getg().m.curg + curg.sig = SIGSEGV + curg.sigcode0 = 0 + curg.sigcode1 = 0 + curg.sigpc = uintptr(pc) + pc = uint32(funcPC(_sfloatpanic)) + break + } + pc += 4 * uint32(skip) + } + if first { + print("sfloat2 ", pc, " ", hex(*(*uint32)(unsafe.Pointer(uintptr(pc)))), "\n") + fabort() // not ok to fail first instruction + } + return pc +} diff --git a/src/runtime/sqrt.go b/src/runtime/sqrt.go index 34a8c3806..e3a27014b 100644 --- a/src/runtime/sqrt.go +++ b/src/runtime/sqrt.go @@ -86,9 +86,6 @@ import "unsafe" // Notes: Rounding mode detection omitted. const ( - uvnan = 0x7FF8000000000001 - uvinf = 0x7FF0000000000000 - uvneginf = 0xFFF0000000000000 mask = 0x7FF shift = 64 - 11 - 1 bias = 1023 @@ -104,7 +101,7 @@ func sqrt(x float64) float64 { case x == 0 || x != x || x > maxFloat64: return x case x < 0: - return nan + return nan() } ix := float64bits(x) // normalize x @@ -144,7 +141,3 @@ func sqrt(x float64) float64 { ix = q>>1 + uint64(exp-1+bias)<<shift // significand + biased exponent return float64frombits(ix) } - -func sqrtC(f float64, r *float64) { - *r = sqrt(f) -} diff --git a/src/runtime/stack.c b/src/runtime/stack.c deleted file mode 100644 index 072bc242b..000000000 --- a/src/runtime/stack.c +++ /dev/null @@ -1,897 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "stack.h" -#include "funcdata.h" -#include "typekind.h" -#include "type.h" -#include "race.h" -#include "mgc0.h" -#include "textflag.h" - -enum -{ - // StackDebug == 0: no logging - // == 1: logging of per-stack operations - // == 2: logging of per-frame operations - // == 3: logging of per-word updates - // == 4: logging of per-word reads - StackDebug = 0, - StackFromSystem = 0, // allocate stacks from system memory instead of the heap - StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free - StackPoisonCopy = 0, // fill stack that should not be accessed with garbage, to detect bad dereferences during copy - - StackCache = 1, -}; - -// Global pool of spans that have free stacks. -// Stacks are assigned an order according to size. -// order = log_2(size/FixedStack) -// There is a free list for each order. -MSpan runtime·stackpool[NumStackOrders]; -Mutex runtime·stackpoolmu; -// TODO: one lock per order? - -static Stack stackfreequeue; - -void -runtime·stackinit(void) -{ - int32 i; - - if((StackCacheSize & PageMask) != 0) - runtime·throw("cache size must be a multiple of page size"); - - for(i = 0; i < NumStackOrders; i++) - runtime·MSpanList_Init(&runtime·stackpool[i]); -} - -// Allocates a stack from the free pool. Must be called with -// stackpoolmu held. -static MLink* -poolalloc(uint8 order) -{ - MSpan *list; - MSpan *s; - MLink *x; - uintptr i; - - list = &runtime·stackpool[order]; - s = list->next; - if(s == list) { - // no free stacks. Allocate another span worth. - s = runtime·MHeap_AllocStack(&runtime·mheap, StackCacheSize >> PageShift); - if(s == nil) - runtime·throw("out of memory"); - if(s->ref != 0) - runtime·throw("bad ref"); - if(s->freelist != nil) - runtime·throw("bad freelist"); - for(i = 0; i < StackCacheSize; i += FixedStack << order) { - x = (MLink*)((s->start << PageShift) + i); - x->next = s->freelist; - s->freelist = x; - } - runtime·MSpanList_Insert(list, s); - } - x = s->freelist; - if(x == nil) - runtime·throw("span has no free stacks"); - s->freelist = x->next; - s->ref++; - if(s->freelist == nil) { - // all stacks in s are allocated. - runtime·MSpanList_Remove(s); - } - return x; -} - -// Adds stack x to the free pool. Must be called with stackpoolmu held. -static void -poolfree(MLink *x, uint8 order) -{ - MSpan *s; - - s = runtime·MHeap_Lookup(&runtime·mheap, x); - if(s->state != MSpanStack) - runtime·throw("freeing stack not in a stack span"); - if(s->freelist == nil) { - // s will now have a free stack - runtime·MSpanList_Insert(&runtime·stackpool[order], s); - } - x->next = s->freelist; - s->freelist = x; - s->ref--; - if(s->ref == 0) { - // span is completely free - return to heap - runtime·MSpanList_Remove(s); - s->freelist = nil; - runtime·MHeap_FreeStack(&runtime·mheap, s); - } -} - -// stackcacherefill/stackcacherelease implement a global pool of stack segments. -// The pool is required to prevent unlimited growth of per-thread caches. -static void -stackcacherefill(MCache *c, uint8 order) -{ - MLink *x, *list; - uintptr size; - - if(StackDebug >= 1) - runtime·printf("stackcacherefill order=%d\n", order); - - // Grab some stacks from the global cache. - // Grab half of the allowed capacity (to prevent thrashing). - list = nil; - size = 0; - runtime·lock(&runtime·stackpoolmu); - while(size < StackCacheSize/2) { - x = poolalloc(order); - x->next = list; - list = x; - size += FixedStack << order; - } - runtime·unlock(&runtime·stackpoolmu); - - c->stackcache[order].list = list; - c->stackcache[order].size = size; -} - -static void -stackcacherelease(MCache *c, uint8 order) -{ - MLink *x, *y; - uintptr size; - - if(StackDebug >= 1) - runtime·printf("stackcacherelease order=%d\n", order); - x = c->stackcache[order].list; - size = c->stackcache[order].size; - runtime·lock(&runtime·stackpoolmu); - while(size > StackCacheSize/2) { - y = x->next; - poolfree(x, order); - x = y; - size -= FixedStack << order; - } - runtime·unlock(&runtime·stackpoolmu); - c->stackcache[order].list = x; - c->stackcache[order].size = size; -} - -void -runtime·stackcache_clear(MCache *c) -{ - uint8 order; - MLink *x, *y; - - if(StackDebug >= 1) - runtime·printf("stackcache clear\n"); - runtime·lock(&runtime·stackpoolmu); - for(order = 0; order < NumStackOrders; order++) { - x = c->stackcache[order].list; - while(x != nil) { - y = x->next; - poolfree(x, order); - x = y; - } - c->stackcache[order].list = nil; - c->stackcache[order].size = 0; - } - runtime·unlock(&runtime·stackpoolmu); -} - -Stack -runtime·stackalloc(uint32 n) -{ - uint8 order; - uint32 n2; - void *v; - MLink *x; - MSpan *s; - MCache *c; - - // Stackalloc must be called on scheduler stack, so that we - // never try to grow the stack during the code that stackalloc runs. - // Doing so would cause a deadlock (issue 1547). - if(g != g->m->g0) - runtime·throw("stackalloc not on scheduler stack"); - if((n & (n-1)) != 0) - runtime·throw("stack size not a power of 2"); - if(StackDebug >= 1) - runtime·printf("stackalloc %d\n", n); - - if(runtime·debug.efence || StackFromSystem) { - v = runtime·sysAlloc(ROUND(n, PageSize), &mstats.stacks_sys); - if(v == nil) - runtime·throw("out of memory (stackalloc)"); - return (Stack){(uintptr)v, (uintptr)v+n}; - } - - // Small stacks are allocated with a fixed-size free-list allocator. - // If we need a stack of a bigger size, we fall back on allocating - // a dedicated span. - if(StackCache && n < FixedStack << NumStackOrders && n < StackCacheSize) { - order = 0; - n2 = n; - while(n2 > FixedStack) { - order++; - n2 >>= 1; - } - c = g->m->mcache; - if(c == nil || g->m->gcing || g->m->helpgc) { - // c == nil can happen in the guts of exitsyscall or - // procresize. Just get a stack from the global pool. - // Also don't touch stackcache during gc - // as it's flushed concurrently. - runtime·lock(&runtime·stackpoolmu); - x = poolalloc(order); - runtime·unlock(&runtime·stackpoolmu); - } else { - x = c->stackcache[order].list; - if(x == nil) { - stackcacherefill(c, order); - x = c->stackcache[order].list; - } - c->stackcache[order].list = x->next; - c->stackcache[order].size -= n; - } - v = (byte*)x; - } else { - s = runtime·MHeap_AllocStack(&runtime·mheap, ROUND(n, PageSize) >> PageShift); - if(s == nil) - runtime·throw("out of memory"); - v = (byte*)(s->start<<PageShift); - } - - if(raceenabled) - runtime·racemalloc(v, n); - if(StackDebug >= 1) - runtime·printf(" allocated %p\n", v); - return (Stack){(uintptr)v, (uintptr)v+n}; -} - -void -runtime·stackfree(Stack stk) -{ - uint8 order; - uintptr n, n2; - MSpan *s; - MLink *x; - MCache *c; - void *v; - - n = stk.hi - stk.lo; - v = (void*)stk.lo; - if(n & (n-1)) - runtime·throw("stack not a power of 2"); - if(StackDebug >= 1) { - runtime·printf("stackfree %p %d\n", v, (int32)n); - runtime·memclr(v, n); // for testing, clobber stack data - } - if(runtime·debug.efence || StackFromSystem) { - if(runtime·debug.efence || StackFaultOnFree) - runtime·SysFault(v, n); - else - runtime·SysFree(v, n, &mstats.stacks_sys); - return; - } - if(StackCache && n < FixedStack << NumStackOrders && n < StackCacheSize) { - order = 0; - n2 = n; - while(n2 > FixedStack) { - order++; - n2 >>= 1; - } - x = (MLink*)v; - c = g->m->mcache; - if(c == nil || g->m->gcing || g->m->helpgc) { - runtime·lock(&runtime·stackpoolmu); - poolfree(x, order); - runtime·unlock(&runtime·stackpoolmu); - } else { - if(c->stackcache[order].size >= StackCacheSize) - stackcacherelease(c, order); - x->next = c->stackcache[order].list; - c->stackcache[order].list = x; - c->stackcache[order].size += n; - } - } else { - s = runtime·MHeap_Lookup(&runtime·mheap, v); - if(s->state != MSpanStack) { - runtime·printf("%p %p\n", s->start<<PageShift, v); - runtime·throw("bad span state"); - } - runtime·MHeap_FreeStack(&runtime·mheap, s); - } -} - -uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real - -static uint8* -mapnames[] = { - (uint8*)"---", - (uint8*)"scalar", - (uint8*)"ptr", - (uint8*)"multi", -}; - -// Stack frame layout -// -// (x86) -// +------------------+ -// | args from caller | -// +------------------+ <- frame->argp -// | return address | -// +------------------+ <- frame->varp -// | locals | -// +------------------+ -// | args to callee | -// +------------------+ <- frame->sp -// -// (arm) -// +------------------+ -// | args from caller | -// +------------------+ <- frame->argp -// | caller's retaddr | -// +------------------+ <- frame->varp -// | locals | -// +------------------+ -// | args to callee | -// +------------------+ -// | return address | -// +------------------+ <- frame->sp - -void runtime·main(void); -void runtime·switchtoM(void(*)(void)); - -typedef struct AdjustInfo AdjustInfo; -struct AdjustInfo { - Stack old; - uintptr delta; // ptr distance from old to new stack (newbase - oldbase) -}; - -// Adjustpointer checks whether *vpp is in the old stack described by adjinfo. -// If so, it rewrites *vpp to point into the new stack. -static void -adjustpointer(AdjustInfo *adjinfo, void *vpp) -{ - byte **pp, *p; - - pp = vpp; - p = *pp; - if(StackDebug >= 4) - runtime·printf(" %p:%p\n", pp, p); - if(adjinfo->old.lo <= (uintptr)p && (uintptr)p < adjinfo->old.hi) { - *pp = p + adjinfo->delta; - if(StackDebug >= 3) - runtime·printf(" adjust ptr %p: %p -> %p\n", pp, p, *pp); - } -} - -// bv describes the memory starting at address scanp. -// Adjust any pointers contained therein. -static void -adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f) -{ - uintptr delta; - int32 num, i; - byte *p, *minp, *maxp; - Type *t; - Itab *tab; - - minp = (byte*)adjinfo->old.lo; - maxp = (byte*)adjinfo->old.hi; - delta = adjinfo->delta; - num = bv->n / BitsPerPointer; - for(i = 0; i < num; i++) { - if(StackDebug >= 4) - runtime·printf(" %p:%s:%p\n", &scanp[i], mapnames[bv->bytedata[i / (8 / BitsPerPointer)] >> (i * BitsPerPointer & 7) & 3], scanp[i]); - switch(bv->bytedata[i / (8 / BitsPerPointer)] >> (i * BitsPerPointer & 7) & 3) { - case BitsDead: - if(runtime·debug.gcdead) - scanp[i] = (byte*)PoisonStack; - break; - case BitsScalar: - break; - case BitsPointer: - p = scanp[i]; - if(f != nil && (byte*)0 < p && (p < (byte*)PageSize && runtime·invalidptr || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) { - // Looks like a junk value in a pointer slot. - // Live analysis wrong? - g->m->traceback = 2; - runtime·printf("runtime: bad pointer in frame %s at %p: %p\n", runtime·funcname(f), &scanp[i], p); - runtime·throw("invalid stack pointer"); - } - if(minp <= p && p < maxp) { - if(StackDebug >= 3) - runtime·printf("adjust ptr %p %s\n", p, runtime·funcname(f)); - scanp[i] = p + delta; - } - break; - case BitsMultiWord: - switch(bv->bytedata[(i+1) / (8 / BitsPerPointer)] >> ((i+1) * BitsPerPointer & 7) & 3) { - default: - runtime·throw("unexpected garbage collection bits"); - case BitsEface: - t = (Type*)scanp[i]; - if(t != nil && ((t->kind & KindDirectIface) == 0 || (t->kind & KindNoPointers) == 0)) { - p = scanp[i+1]; - if(minp <= p && p < maxp) { - if(StackDebug >= 3) - runtime·printf("adjust eface %p\n", p); - if(t->size > PtrSize) // currently we always allocate such objects on the heap - runtime·throw("large interface value found on stack"); - scanp[i+1] = p + delta; - } - } - i++; - break; - case BitsIface: - tab = (Itab*)scanp[i]; - if(tab != nil) { - t = tab->type; - //runtime·printf(" type=%p\n", t); - if((t->kind & KindDirectIface) == 0 || (t->kind & KindNoPointers) == 0) { - p = scanp[i+1]; - if(minp <= p && p < maxp) { - if(StackDebug >= 3) - runtime·printf("adjust iface %p\n", p); - if(t->size > PtrSize) // currently we always allocate such objects on the heap - runtime·throw("large interface value found on stack"); - scanp[i+1] = p + delta; - } - } - } - i++; - break; - } - break; - } - } -} - -// Note: the argument/return area is adjusted by the callee. -static bool -adjustframe(Stkframe *frame, void *arg) -{ - AdjustInfo *adjinfo; - Func *f; - StackMap *stackmap; - int32 pcdata; - BitVector bv; - uintptr targetpc, size, minsize; - - adjinfo = arg; - targetpc = frame->continpc; - if(targetpc == 0) { - // Frame is dead. - return true; - } - f = frame->fn; - if(StackDebug >= 2) - runtime·printf(" adjusting %s frame=[%p,%p] pc=%p continpc=%p\n", runtime·funcname(f), frame->sp, frame->fp, frame->pc, frame->continpc); - if(f->entry == (uintptr)runtime·switchtoM) { - // A special routine at the bottom of stack of a goroutine that does an onM call. - // We will allow it to be copied even though we don't - // have full GC info for it (because it is written in asm). - return true; - } - if(targetpc != f->entry) - targetpc--; - pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); - if(pcdata == -1) - pcdata = 0; // in prologue - - // Adjust local variables if stack frame has been allocated. - size = frame->varp - frame->sp; - if(thechar != '6' && thechar != '8') - minsize = sizeof(uintptr); - else - minsize = 0; - if(size > minsize) { - stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); - if(stackmap == nil || stackmap->n <= 0) { - runtime·printf("runtime: frame %s untyped locals %p+%p\n", runtime·funcname(f), (byte*)(frame->varp-size), size); - runtime·throw("missing stackmap"); - } - // Locals bitmap information, scan just the pointers in locals. - if(pcdata < 0 || pcdata >= stackmap->n) { - // don't know where we are - runtime·printf("runtime: pcdata is %d and %d locals stack map entries for %s (targetpc=%p)\n", - pcdata, stackmap->n, runtime·funcname(f), targetpc); - runtime·throw("bad symbol table"); - } - bv = runtime·stackmapdata(stackmap, pcdata); - size = (bv.n * PtrSize) / BitsPerPointer; - if(StackDebug >= 3) - runtime·printf(" locals\n"); - adjustpointers((byte**)(frame->varp - size), &bv, adjinfo, f); - } - - // Adjust arguments. - if(frame->arglen > 0) { - if(frame->argmap != nil) { - bv = *frame->argmap; - } else { - stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); - if(stackmap == nil || stackmap->n <= 0) { - runtime·printf("runtime: frame %s untyped args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen); - runtime·throw("missing stackmap"); - } - if(pcdata < 0 || pcdata >= stackmap->n) { - // don't know where we are - runtime·printf("runtime: pcdata is %d and %d args stack map entries for %s (targetpc=%p)\n", - pcdata, stackmap->n, runtime·funcname(f), targetpc); - runtime·throw("bad symbol table"); - } - bv = runtime·stackmapdata(stackmap, pcdata); - } - if(StackDebug >= 3) - runtime·printf(" args\n"); - adjustpointers((byte**)frame->argp, &bv, adjinfo, nil); - } - - return true; -} - -static void -adjustctxt(G *gp, AdjustInfo *adjinfo) -{ - adjustpointer(adjinfo, &gp->sched.ctxt); -} - -static void -adjustdefers(G *gp, AdjustInfo *adjinfo) -{ - Defer *d; - bool (*cb)(Stkframe*, void*); - - // Adjust defer argument blocks the same way we adjust active stack frames. - cb = adjustframe; - runtime·tracebackdefers(gp, &cb, adjinfo); - - // Adjust pointers in the Defer structs. - // Defer structs themselves are never on the stack. - for(d = gp->defer; d != nil; d = d->link) { - adjustpointer(adjinfo, &d->fn); - adjustpointer(adjinfo, &d->argp); - adjustpointer(adjinfo, &d->panic); - } -} - -static void -adjustpanics(G *gp, AdjustInfo *adjinfo) -{ - // Panics are on stack and already adjusted. - // Update pointer to head of list in G. - adjustpointer(adjinfo, &gp->panic); -} - -static void -adjustsudogs(G *gp, AdjustInfo *adjinfo) -{ - SudoG *s; - - // the data elements pointed to by a SudoG structure - // might be in the stack. - for(s = gp->waiting; s != nil; s = s->waitlink) { - adjustpointer(adjinfo, &s->elem); - adjustpointer(adjinfo, &s->selectdone); - } -} - -// Copies gp's stack to a new stack of a different size. -static void -copystack(G *gp, uintptr newsize) -{ - Stack old, new; - uintptr used; - AdjustInfo adjinfo; - uint32 oldstatus; - bool (*cb)(Stkframe*, void*); - byte *p, *ep; - - if(gp->syscallsp != 0) - runtime·throw("stack growth not allowed in system call"); - old = gp->stack; - if(old.lo == 0) - runtime·throw("nil stackbase"); - used = old.hi - gp->sched.sp; - - // allocate new stack - new = runtime·stackalloc(newsize); - if(StackPoisonCopy) { - p = (byte*)new.lo; - ep = (byte*)new.hi; - while(p < ep) - *p++ = 0xfd; - } - - if(StackDebug >= 1) - runtime·printf("copystack gp=%p [%p %p %p]/%d -> [%p %p %p]/%d\n", gp, old.lo, old.hi-used, old.hi, (int32)(old.hi-old.lo), new.lo, new.hi-used, new.hi, (int32)newsize); - - // adjust pointers in the to-be-copied frames - adjinfo.old = old; - adjinfo.delta = new.hi - old.hi; - cb = adjustframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, 0); - - // adjust other miscellaneous things that have pointers into stacks. - adjustctxt(gp, &adjinfo); - adjustdefers(gp, &adjinfo); - adjustpanics(gp, &adjinfo); - adjustsudogs(gp, &adjinfo); - - // copy the stack to the new location - if(StackPoisonCopy) { - p = (byte*)new.lo; - ep = (byte*)new.hi; - while(p < ep) - *p++ = 0xfb; - } - runtime·memmove((byte*)new.hi - used, (byte*)old.hi - used, used); - - oldstatus = runtime·readgstatus(gp); - oldstatus &= ~Gscan; - if(oldstatus == Gwaiting || oldstatus == Grunnable) - runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable - else - runtime·throw("copystack: bad status, not Gwaiting or Grunnable"); - - // Swap out old stack for new one - gp->stack = new; - gp->stackguard0 = new.lo + StackGuard; // NOTE: might clobber a preempt request - gp->sched.sp = new.hi - used; - - runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Gwaiting or Grunnable - - // free old stack - if(StackPoisonCopy) { - p = (byte*)old.lo; - ep = (byte*)old.hi; - while(p < ep) - *p++ = 0xfc; - } - if(newsize > old.hi-old.lo) { - // growing, free stack immediately - runtime·stackfree(old); - } else { - // shrinking, queue up free operation. We can't actually free the stack - // just yet because we might run into the following situation: - // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer - // 2) The stack that pointer points to is shrunk - // 3) The old stack is freed - // 4) The containing span is marked free - // 5) GC attempts to mark the SudoG.elem pointer. The marking fails because - // the pointer looks like a pointer into a free span. - // By not freeing, we prevent step #4 until GC is done. - runtime·lock(&runtime·stackpoolmu); - *(Stack*)old.lo = stackfreequeue; - stackfreequeue = old; - runtime·unlock(&runtime·stackpoolmu); - } -} - -// round x up to a power of 2. -int32 -runtime·round2(int32 x) -{ - int32 s; - - s = 0; - while((1 << s) < x) - s++; - return 1 << s; -} - -// Called from runtime·morestack when more stack is needed. -// Allocate larger stack and relocate to new stack. -// Stack growth is multiplicative, for constant amortized cost. -// -// g->atomicstatus will be Grunning or Gscanrunning upon entry. -// If the GC is trying to stop this g then it will set preemptscan to true. -void -runtime·newstack(void) -{ - int32 oldsize, newsize; - uintptr sp; - G *gp; - Gobuf morebuf; - - if(g->m->morebuf.g->stackguard0 == (uintptr)StackFork) - runtime·throw("stack growth after fork"); - if(g->m->morebuf.g != g->m->curg) { - runtime·printf("runtime: newstack called from g=%p\n" - "\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n", - g->m->morebuf.g, g->m, g->m->curg, g->m->g0, g->m->gsignal); - morebuf = g->m->morebuf; - runtime·traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g); - runtime·throw("runtime: wrong goroutine in newstack"); - } - if(g->m->curg->throwsplit) - runtime·throw("runtime: stack split at bad time"); - - // The goroutine must be executing in order to call newstack, - // so it must be Grunning or Gscanrunning. - - gp = g->m->curg; - morebuf = g->m->morebuf; - g->m->morebuf.pc = (uintptr)nil; - g->m->morebuf.lr = (uintptr)nil; - g->m->morebuf.sp = (uintptr)nil; - g->m->morebuf.g = (G*)nil; - - runtime·casgstatus(gp, Grunning, Gwaiting); - gp->waitreason = runtime·gostringnocopy((byte*)"stack growth"); - - runtime·rewindmorestack(&gp->sched); - - if(gp->stack.lo == 0) - runtime·throw("missing stack in newstack"); - sp = gp->sched.sp; - if(thechar == '6' || thechar == '8') { - // The call to morestack cost a word. - sp -= sizeof(uintreg); - } - if(StackDebug >= 1 || sp < gp->stack.lo) { - runtime·printf("runtime: newstack sp=%p stack=[%p, %p]\n" - "\tmorebuf={pc:%p sp:%p lr:%p}\n" - "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n", - sp, gp->stack.lo, gp->stack.hi, - g->m->morebuf.pc, g->m->morebuf.sp, g->m->morebuf.lr, - gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt); - } - if(sp < gp->stack.lo) { - runtime·printf("runtime: gp=%p, gp->status=%d\n ", (void*)gp, runtime·readgstatus(gp)); - runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stack.lo); - runtime·throw("runtime: split stack overflow"); - } - - if(gp->stackguard0 == (uintptr)StackPreempt) { - if(gp == g->m->g0) - runtime·throw("runtime: preempt g0"); - if(g->m->p == nil && g->m->locks == 0) - runtime·throw("runtime: g is running but p is not"); - if(gp->preemptscan) { - runtime·gcphasework(gp); - runtime·casgstatus(gp, Gwaiting, Grunning); - gp->stackguard0 = gp->stack.lo + StackGuard; - gp->preempt = false; - gp->preemptscan = false; // Tells the GC premption was successful. - runtime·gogo(&gp->sched); // never return - } - - // Be conservative about where we preempt. - // We are interested in preempting user Go code, not runtime code. - if(g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) { - // Let the goroutine keep running for now. - // gp->preempt is set, so it will be preempted next time. - gp->stackguard0 = gp->stack.lo + StackGuard; - runtime·casgstatus(gp, Gwaiting, Grunning); - runtime·gogo(&gp->sched); // never return - } - // Act like goroutine called runtime.Gosched. - runtime·casgstatus(gp, Gwaiting, Grunning); - runtime·gosched_m(gp); // never return - } - - // Allocate a bigger segment and move the stack. - oldsize = gp->stack.hi - gp->stack.lo; - newsize = oldsize * 2; - if(newsize > runtime·maxstacksize) { - runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); - runtime·throw("stack overflow"); - } - - // Note that the concurrent GC might be scanning the stack as we try to replace it. - // copystack takes care of the appropriate coordination with the stack scanner. - copystack(gp, newsize); - if(StackDebug >= 1) - runtime·printf("stack grow done\n"); - runtime·casgstatus(gp, Gwaiting, Grunning); - runtime·gogo(&gp->sched); -} - -#pragma textflag NOSPLIT -void -runtime·nilfunc(void) -{ - *(byte*)0 = 0; -} - -// adjust Gobuf as if it executed a call to fn -// and then did an immediate gosave. -void -runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) -{ - void *fn; - - if(fv != nil) - fn = fv->fn; - else - fn = runtime·nilfunc; - runtime·gostartcall(gobuf, fn, fv); -} - -// Maybe shrink the stack being used by gp. -// Called at garbage collection time. -void -runtime·shrinkstack(G *gp) -{ - uintptr used, oldsize, newsize; - - if(runtime·readgstatus(gp) == Gdead) { - if(gp->stack.lo != 0) { - // Free whole stack - it will get reallocated - // if G is used again. - runtime·stackfree(gp->stack); - gp->stack.lo = 0; - gp->stack.hi = 0; - } - return; - } - if(gp->stack.lo == 0) - runtime·throw("missing stack in shrinkstack"); - - oldsize = gp->stack.hi - gp->stack.lo; - newsize = oldsize / 2; - if(newsize < FixedStack) - return; // don't shrink below the minimum-sized stack - used = gp->stack.hi - gp->sched.sp; - if(used >= oldsize / 4) - return; // still using at least 1/4 of the segment. - - // We can't copy the stack if we're in a syscall. - // The syscall might have pointers into the stack. - if(gp->syscallsp != 0) - return; - -#ifdef GOOS_windows - if(gp->m != nil && gp->m->libcallsp != 0) - return; -#endif - if(StackDebug > 0) - runtime·printf("shrinking stack %D->%D\n", (uint64)oldsize, (uint64)newsize); - copystack(gp, newsize); -} - -// Do any delayed stack freeing that was queued up during GC. -void -runtime·shrinkfinish(void) -{ - Stack s, t; - - runtime·lock(&runtime·stackpoolmu); - s = stackfreequeue; - stackfreequeue = (Stack){0,0}; - runtime·unlock(&runtime·stackpoolmu); - while(s.lo != 0) { - t = *(Stack*)s.lo; - runtime·stackfree(s); - s = t; - } -} - -static void badc(void); - -#pragma textflag NOSPLIT -void -runtime·morestackc(void) -{ - void (*fn)(void); - - fn = badc; - runtime·onM(&fn); -} - -static void -badc(void) -{ - runtime·throw("attempt to execute C code on Go stack"); -} diff --git a/src/runtime/stack.go b/src/runtime/stack.go deleted file mode 100644 index f1b7d32d2..000000000 --- a/src/runtime/stack.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const ( - // Goroutine preemption request. - // Stored into g->stackguard0 to cause split stack check failure. - // Must be greater than any real sp. - // 0xfffffade in hex. - stackPreempt = ^uintptr(1313) -) diff --git a/src/runtime/stack.h b/src/runtime/stack.h index f97dc4ed8..0099d05c2 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -2,117 +2,24 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* -Stack layout parameters. -Included both by runtime (compiled via 6c) and linkers (compiled via gcc). - -The per-goroutine g->stackguard is set to point StackGuard bytes -above the bottom of the stack. Each function compares its stack -pointer against g->stackguard to check for overflow. To cut one -instruction from the check sequence for functions with tiny frames, -the stack is allowed to protrude StackSmall bytes below the stack -guard. Functions with large frames don't bother with the check and -always call morestack. The sequences are (for amd64, others are -similar): - - guard = g->stackguard - frame = function's stack frame size - argsize = size of function arguments (call + return) - - stack frame size <= StackSmall: - CMPQ guard, SP - JHI 3(PC) - MOVQ m->morearg, $(argsize << 32) - CALL morestack(SB) - - stack frame size > StackSmall but < StackBig - LEAQ (frame-StackSmall)(SP), R0 - CMPQ guard, R0 - JHI 3(PC) - MOVQ m->morearg, $(argsize << 32) - CALL morestack(SB) - - stack frame size >= StackBig: - MOVQ m->morearg, $((argsize << 32) | frame) - CALL morestack(SB) - -The bottom StackGuard - StackSmall bytes are important: there has -to be enough room to execute functions that refuse to check for -stack overflow, either because they need to be adjacent to the -actual caller's frame (deferproc) or because they handle the imminent -stack overflow (morestack). - -For example, deferproc might call malloc, which does one of the -above checks (without allocating a full frame), which might trigger -a call to morestack. This sequence needs to fit in the bottom -section of the stack. On amd64, morestack's frame is 40 bytes, and -deferproc's frame is 56 bytes. That fits well within the -StackGuard - StackSmall bytes at the bottom. -The linkers explore all possible call traces involving non-splitting -functions to make sure that this limit cannot be violated. - */ +// For the linkers. Must match Go definitions. +// TODO(rsc): Share Go definitions with linkers directly. enum { - // StackSystem is a number of additional bytes to add - // to each stack below the usual guard area for OS-specific - // purposes like signal handling. Used on Windows and on - // Plan 9 because they do not use a separate stack. #ifdef GOOS_windows StackSystem = 512 * sizeof(uintptr), #else #ifdef GOOS_plan9 - // The size of the note handler frame varies among architectures, - // but 512 bytes should be enough for every implementation. StackSystem = 512, #else StackSystem = 0, #endif // Plan 9 #endif // Windows - // The minimum size of stack used by Go code - StackMin = 2048, - - // The minimum stack size to allocate. - // The hackery here rounds FixedStack0 up to a power of 2. - FixedStack0 = StackMin + StackSystem, - FixedStack1 = FixedStack0 - 1, - FixedStack2 = FixedStack1 | (FixedStack1 >> 1), - FixedStack3 = FixedStack2 | (FixedStack2 >> 2), - FixedStack4 = FixedStack3 | (FixedStack3 >> 4), - FixedStack5 = FixedStack4 | (FixedStack4 >> 8), - FixedStack6 = FixedStack5 | (FixedStack5 >> 16), - FixedStack = FixedStack6 + 1, - - // Functions that need frames bigger than this use an extra - // instruction to do the stack split check, to avoid overflow - // in case SP - framesize wraps below zero. - // This value can be no bigger than the size of the unmapped - // space at zero. StackBig = 4096, - - // The stack guard is a pointer this many bytes above the - // bottom of the stack. StackGuard = 512 + StackSystem, - - // After a stack split check the SP is allowed to be this - // many bytes below the stack guard. This saves an instruction - // in the checking sequence for tiny frames. StackSmall = 128, - - // The maximum number of bytes that a chain of NOSPLIT - // functions can use. StackLimit = StackGuard - StackSystem - StackSmall, }; -// Goroutine preemption request. -// Stored into g->stackguard0 to cause split stack check failure. -// Must be greater than any real sp. -// 0xfffffade in hex. #define StackPreempt ((uint64)-1314) -/*c2go -enum -{ - StackPreempt = -1314, -}; -*/ -#define StackFork ((uint64)-1234) diff --git a/src/runtime/stack1.go b/src/runtime/stack1.go new file mode 100644 index 000000000..ad83e5895 --- /dev/null +++ b/src/runtime/stack1.go @@ -0,0 +1,807 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + // StackDebug == 0: no logging + // == 1: logging of per-stack operations + // == 2: logging of per-frame operations + // == 3: logging of per-word updates + // == 4: logging of per-word reads + stackDebug = 0 + stackFromSystem = 0 // allocate stacks from system memory instead of the heap + stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free + stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy + + stackCache = 1 +) + +const ( + uintptrMask = 1<<(8*ptrSize) - 1 + poisonGC = uintptrMask & 0xf969696969696969 + poisonStack = uintptrMask & 0x6868686868686868 + + // Goroutine preemption request. + // Stored into g->stackguard0 to cause split stack check failure. + // Must be greater than any real sp. + // 0xfffffade in hex. + stackPreempt = uintptrMask & -1314 + + // Thread is forking. + // Stored into g->stackguard0 to cause split stack check failure. + // Must be greater than any real sp. + stackFork = uintptrMask & -1234 +) + +// Global pool of spans that have free stacks. +// Stacks are assigned an order according to size. +// order = log_2(size/FixedStack) +// There is a free list for each order. +// TODO: one lock per order? +var stackpool [_NumStackOrders]mspan +var stackpoolmu mutex + +var stackfreequeue stack + +func stackinit() { + if _StackCacheSize&_PageMask != 0 { + gothrow("cache size must be a multiple of page size") + } + for i := range stackpool { + mSpanList_Init(&stackpool[i]) + } +} + +// Allocates a stack from the free pool. Must be called with +// stackpoolmu held. +func stackpoolalloc(order uint8) *mlink { + list := &stackpool[order] + s := list.next + if s == list { + // no free stacks. Allocate another span worth. + s = mHeap_AllocStack(&mheap_, _StackCacheSize>>_PageShift) + if s == nil { + gothrow("out of memory") + } + if s.ref != 0 { + gothrow("bad ref") + } + if s.freelist != nil { + gothrow("bad freelist") + } + for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order { + x := (*mlink)(unsafe.Pointer(uintptr(s.start)<<_PageShift + i)) + x.next = s.freelist + s.freelist = x + } + mSpanList_Insert(list, s) + } + x := s.freelist + if x == nil { + gothrow("span has no free stacks") + } + s.freelist = x.next + s.ref++ + if s.freelist == nil { + // all stacks in s are allocated. + mSpanList_Remove(s) + } + return x +} + +// Adds stack x to the free pool. Must be called with stackpoolmu held. +func stackpoolfree(x *mlink, order uint8) { + s := mHeap_Lookup(&mheap_, (unsafe.Pointer)(x)) + if s.state != _MSpanStack { + gothrow("freeing stack not in a stack span") + } + if s.freelist == nil { + // s will now have a free stack + mSpanList_Insert(&stackpool[order], s) + } + x.next = s.freelist + s.freelist = x + s.ref-- + if s.ref == 0 { + // span is completely free - return to heap + mSpanList_Remove(s) + s.freelist = nil + mHeap_FreeStack(&mheap_, s) + } +} + +// stackcacherefill/stackcacherelease implement a global pool of stack segments. +// The pool is required to prevent unlimited growth of per-thread caches. +func stackcacherefill(c *mcache, order uint8) { + if stackDebug >= 1 { + print("stackcacherefill order=", order, "\n") + } + + // Grab some stacks from the global cache. + // Grab half of the allowed capacity (to prevent thrashing). + var list *mlink + var size uintptr + lock(&stackpoolmu) + for size < _StackCacheSize/2 { + x := stackpoolalloc(order) + x.next = list + list = x + size += _FixedStack << order + } + unlock(&stackpoolmu) + c.stackcache[order].list = list + c.stackcache[order].size = size +} + +func stackcacherelease(c *mcache, order uint8) { + if stackDebug >= 1 { + print("stackcacherelease order=", order, "\n") + } + x := c.stackcache[order].list + size := c.stackcache[order].size + lock(&stackpoolmu) + for size > _StackCacheSize/2 { + y := x.next + stackpoolfree(x, order) + x = y + size -= _FixedStack << order + } + unlock(&stackpoolmu) + c.stackcache[order].list = x + c.stackcache[order].size = size +} + +func stackcache_clear(c *mcache) { + if stackDebug >= 1 { + print("stackcache clear\n") + } + lock(&stackpoolmu) + for order := uint8(0); order < _NumStackOrders; order++ { + x := c.stackcache[order].list + for x != nil { + y := x.next + stackpoolfree(x, order) + x = y + } + c.stackcache[order].list = nil + c.stackcache[order].size = 0 + } + unlock(&stackpoolmu) +} + +func stackalloc(n uint32) stack { + // Stackalloc must be called on scheduler stack, so that we + // never try to grow the stack during the code that stackalloc runs. + // Doing so would cause a deadlock (issue 1547). + thisg := getg() + if thisg != thisg.m.g0 { + gothrow("stackalloc not on scheduler stack") + } + if n&(n-1) != 0 { + gothrow("stack size not a power of 2") + } + if stackDebug >= 1 { + print("stackalloc ", n, "\n") + } + + if debug.efence != 0 || stackFromSystem != 0 { + v := sysAlloc(round(uintptr(n), _PageSize), &memstats.stacks_sys) + if v == nil { + gothrow("out of memory (stackalloc)") + } + return stack{uintptr(v), uintptr(v) + uintptr(n)} + } + + // Small stacks are allocated with a fixed-size free-list allocator. + // If we need a stack of a bigger size, we fall back on allocating + // a dedicated span. + var v unsafe.Pointer + if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize { + order := uint8(0) + n2 := n + for n2 > _FixedStack { + order++ + n2 >>= 1 + } + var x *mlink + c := thisg.m.mcache + if c == nil || thisg.m.gcing != 0 || thisg.m.helpgc != 0 { + // c == nil can happen in the guts of exitsyscall or + // procresize. Just get a stack from the global pool. + // Also don't touch stackcache during gc + // as it's flushed concurrently. + lock(&stackpoolmu) + x = stackpoolalloc(order) + unlock(&stackpoolmu) + } else { + x = c.stackcache[order].list + if x == nil { + stackcacherefill(c, order) + x = c.stackcache[order].list + } + c.stackcache[order].list = x.next + c.stackcache[order].size -= uintptr(n) + } + v = (unsafe.Pointer)(x) + } else { + s := mHeap_AllocStack(&mheap_, round(uintptr(n), _PageSize)>>_PageShift) + if s == nil { + gothrow("out of memory") + } + v = (unsafe.Pointer)(s.start << _PageShift) + } + + if raceenabled { + racemalloc(v, uintptr(n)) + } + if stackDebug >= 1 { + print(" allocated ", v, "\n") + } + return stack{uintptr(v), uintptr(v) + uintptr(n)} +} + +func stackfree(stk stack) { + gp := getg() + n := stk.hi - stk.lo + v := (unsafe.Pointer)(stk.lo) + if n&(n-1) != 0 { + gothrow("stack not a power of 2") + } + if stackDebug >= 1 { + println("stackfree", v, n) + memclr(v, n) // for testing, clobber stack data + } + if debug.efence != 0 || stackFromSystem != 0 { + if debug.efence != 0 || stackFaultOnFree != 0 { + sysFault(v, n) + } else { + sysFree(v, n, &memstats.stacks_sys) + } + return + } + if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize { + order := uint8(0) + n2 := n + for n2 > _FixedStack { + order++ + n2 >>= 1 + } + x := (*mlink)(v) + c := gp.m.mcache + if c == nil || gp.m.gcing != 0 || gp.m.helpgc != 0 { + lock(&stackpoolmu) + stackpoolfree(x, order) + unlock(&stackpoolmu) + } else { + if c.stackcache[order].size >= _StackCacheSize { + stackcacherelease(c, order) + } + x.next = c.stackcache[order].list + c.stackcache[order].list = x + c.stackcache[order].size += n + } + } else { + s := mHeap_Lookup(&mheap_, v) + if s.state != _MSpanStack { + println(hex(s.start<<_PageShift), v) + gothrow("bad span state") + } + mHeap_FreeStack(&mheap_, s) + } +} + +var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real + +var mapnames = []string{ + _BitsDead: "---", + _BitsScalar: "scalar", + _BitsPointer: "ptr", +} + +// Stack frame layout +// +// (x86) +// +------------------+ +// | args from caller | +// +------------------+ <- frame->argp +// | return address | +// +------------------+ <- frame->varp +// | locals | +// +------------------+ +// | args to callee | +// +------------------+ <- frame->sp +// +// (arm) +// +------------------+ +// | args from caller | +// +------------------+ <- frame->argp +// | caller's retaddr | +// +------------------+ <- frame->varp +// | locals | +// +------------------+ +// | args to callee | +// +------------------+ +// | return address | +// +------------------+ <- frame->sp + +type adjustinfo struct { + old stack + delta uintptr // ptr distance from old to new stack (newbase - oldbase) +} + +// Adjustpointer checks whether *vpp is in the old stack described by adjinfo. +// If so, it rewrites *vpp to point into the new stack. +func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) { + pp := (*unsafe.Pointer)(vpp) + p := *pp + if stackDebug >= 4 { + print(" ", pp, ":", p, "\n") + } + if adjinfo.old.lo <= uintptr(p) && uintptr(p) < adjinfo.old.hi { + *pp = add(p, adjinfo.delta) + if stackDebug >= 3 { + print(" adjust ptr ", pp, ":", p, " -> ", *pp, "\n") + } + } +} + +type gobitvector struct { + n uintptr + bytedata []uint8 +} + +func gobv(bv bitvector) gobitvector { + return gobitvector{ + uintptr(bv.n), + (*[1 << 30]byte)(unsafe.Pointer(bv.bytedata))[:(bv.n+7)/8], + } +} + +func ptrbits(bv *gobitvector, i uintptr) uint8 { + return (bv.bytedata[i/4] >> ((i & 3) * 2)) & 3 +} + +// bv describes the memory starting at address scanp. +// Adjust any pointers contained therein. +func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f *_func) { + bv := gobv(*cbv) + minp := adjinfo.old.lo + maxp := adjinfo.old.hi + delta := adjinfo.delta + num := uintptr(bv.n / _BitsPerPointer) + for i := uintptr(0); i < num; i++ { + if stackDebug >= 4 { + print(" ", add(scanp, i*ptrSize), ":", mapnames[ptrbits(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*ptrSize))), " # ", i, " ", bv.bytedata[i/4], "\n") + } + switch ptrbits(&bv, i) { + default: + gothrow("unexpected pointer bits") + case _BitsDead: + if debug.gcdead != 0 { + *(*unsafe.Pointer)(add(scanp, i*ptrSize)) = unsafe.Pointer(uintptr(poisonStack)) + } + case _BitsScalar: + // ok + case _BitsPointer: + p := *(*unsafe.Pointer)(add(scanp, i*ptrSize)) + up := uintptr(p) + if f != nil && 0 < up && up < _PageSize && invalidptr != 0 || up == poisonGC || up == poisonStack { + // Looks like a junk value in a pointer slot. + // Live analysis wrong? + getg().m.traceback = 2 + print("runtime: bad pointer in frame ", gofuncname(f), " at ", add(scanp, i*ptrSize), ": ", p, "\n") + gothrow("invalid stack pointer") + } + if minp <= up && up < maxp { + if stackDebug >= 3 { + print("adjust ptr ", p, " ", gofuncname(f), "\n") + } + *(*unsafe.Pointer)(add(scanp, i*ptrSize)) = unsafe.Pointer(up + delta) + } + } + } +} + +// Note: the argument/return area is adjusted by the callee. +func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { + adjinfo := (*adjustinfo)(arg) + targetpc := frame.continpc + if targetpc == 0 { + // Frame is dead. + return true + } + f := frame.fn + if stackDebug >= 2 { + print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") + } + if f.entry == systemstack_switchPC { + // A special routine at the bottom of stack of a goroutine that does an systemstack call. + // We will allow it to be copied even though we don't + // have full GC info for it (because it is written in asm). + return true + } + if targetpc != f.entry { + targetpc-- + } + pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc) + if pcdata == -1 { + pcdata = 0 // in prologue + } + + // Adjust local variables if stack frame has been allocated. + size := frame.varp - frame.sp + var minsize uintptr + if thechar != '6' && thechar != '8' { + minsize = ptrSize + } else { + minsize = 0 + } + if size > minsize { + var bv bitvector + stackmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) + if stackmap == nil || stackmap.n <= 0 { + print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n") + gothrow("missing stackmap") + } + // Locals bitmap information, scan just the pointers in locals. + if pcdata < 0 || pcdata >= stackmap.n { + // don't know where we are + print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n") + gothrow("bad symbol table") + } + bv = stackmapdata(stackmap, pcdata) + size = (uintptr(bv.n) * ptrSize) / _BitsPerPointer + if stackDebug >= 3 { + print(" locals ", pcdata, "/", stackmap.n, " ", size/ptrSize, " words ", bv.bytedata, "\n") + } + adjustpointers(unsafe.Pointer(frame.varp-size), &bv, adjinfo, f) + } + + // Adjust arguments. + if frame.arglen > 0 { + var bv bitvector + if frame.argmap != nil { + bv = *frame.argmap + } else { + stackmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps)) + if stackmap == nil || stackmap.n <= 0 { + print("runtime: frame ", funcname(f), " untyped args ", frame.argp, "+", uintptr(frame.arglen), "\n") + gothrow("missing stackmap") + } + if pcdata < 0 || pcdata >= stackmap.n { + // don't know where we are + print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n") + gothrow("bad symbol table") + } + bv = stackmapdata(stackmap, pcdata) + } + if stackDebug >= 3 { + print(" args\n") + } + adjustpointers(unsafe.Pointer(frame.argp), &bv, adjinfo, nil) + } + return true +} + +func adjustctxt(gp *g, adjinfo *adjustinfo) { + adjustpointer(adjinfo, (unsafe.Pointer)(&gp.sched.ctxt)) +} + +func adjustdefers(gp *g, adjinfo *adjustinfo) { + // Adjust defer argument blocks the same way we adjust active stack frames. + tracebackdefers(gp, adjustframe, noescape(unsafe.Pointer(adjinfo))) + + // Adjust pointers in the Defer structs. + // Defer structs themselves are never on the stack. + for d := gp._defer; d != nil; d = d.link { + adjustpointer(adjinfo, (unsafe.Pointer)(&d.fn)) + adjustpointer(adjinfo, (unsafe.Pointer)(&d.argp)) + adjustpointer(adjinfo, (unsafe.Pointer)(&d._panic)) + } +} + +func adjustpanics(gp *g, adjinfo *adjustinfo) { + // Panics are on stack and already adjusted. + // Update pointer to head of list in G. + adjustpointer(adjinfo, (unsafe.Pointer)(&gp._panic)) +} + +func adjustsudogs(gp *g, adjinfo *adjustinfo) { + // the data elements pointed to by a SudoG structure + // might be in the stack. + for s := gp.waiting; s != nil; s = s.waitlink { + adjustpointer(adjinfo, (unsafe.Pointer)(&s.elem)) + adjustpointer(adjinfo, (unsafe.Pointer)(&s.selectdone)) + } +} + +func fillstack(stk stack, b byte) { + for p := stk.lo; p < stk.hi; p++ { + *(*byte)(unsafe.Pointer(p)) = b + } +} + +// Copies gp's stack to a new stack of a different size. +func copystack(gp *g, newsize uintptr) { + if gp.syscallsp != 0 { + gothrow("stack growth not allowed in system call") + } + old := gp.stack + if old.lo == 0 { + gothrow("nil stackbase") + } + used := old.hi - gp.sched.sp + + // allocate new stack + new := stackalloc(uint32(newsize)) + if stackPoisonCopy != 0 { + fillstack(new, 0xfd) + } + if stackDebug >= 1 { + print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]/", old.hi-old.lo, " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n") + } + + // adjust pointers in the to-be-copied frames + var adjinfo adjustinfo + adjinfo.old = old + adjinfo.delta = new.hi - old.hi + gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, adjustframe, noescape(unsafe.Pointer(&adjinfo)), 0) + + // adjust other miscellaneous things that have pointers into stacks. + adjustctxt(gp, &adjinfo) + adjustdefers(gp, &adjinfo) + adjustpanics(gp, &adjinfo) + adjustsudogs(gp, &adjinfo) + + // copy the stack to the new location + if stackPoisonCopy != 0 { + fillstack(new, 0xfb) + } + memmove(unsafe.Pointer(new.hi-used), unsafe.Pointer(old.hi-used), used) + + oldstatus := readgstatus(gp) + oldstatus &^= _Gscan + if oldstatus == _Gwaiting || oldstatus == _Grunnable { + casgstatus(gp, oldstatus, _Gcopystack) // oldstatus is Gwaiting or Grunnable + } else { + gothrow("copystack: bad status, not Gwaiting or Grunnable") + } + + // Swap out old stack for new one + gp.stack = new + gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request + gp.sched.sp = new.hi - used + + casgstatus(gp, _Gcopystack, oldstatus) // oldstatus is Gwaiting or Grunnable + + // free old stack + if stackPoisonCopy != 0 { + fillstack(old, 0xfc) + } + if newsize > old.hi-old.lo { + // growing, free stack immediately + stackfree(old) + } else { + // shrinking, queue up free operation. We can't actually free the stack + // just yet because we might run into the following situation: + // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer + // 2) The stack that pointer points to is shrunk + // 3) The old stack is freed + // 4) The containing span is marked free + // 5) GC attempts to mark the SudoG.elem pointer. The marking fails because + // the pointer looks like a pointer into a free span. + // By not freeing, we prevent step #4 until GC is done. + lock(&stackpoolmu) + *(*stack)(unsafe.Pointer(old.lo)) = stackfreequeue + stackfreequeue = old + unlock(&stackpoolmu) + } +} + +// round x up to a power of 2. +func round2(x int32) int32 { + s := uint(0) + for 1<<s < x { + s++ + } + return 1 << s +} + +// Called from runtime·morestack when more stack is needed. +// Allocate larger stack and relocate to new stack. +// Stack growth is multiplicative, for constant amortized cost. +// +// g->atomicstatus will be Grunning or Gscanrunning upon entry. +// If the GC is trying to stop this g then it will set preemptscan to true. +func newstack() { + thisg := getg() + // TODO: double check all gp. shouldn't be getg(). + if thisg.m.morebuf.g.stackguard0 == stackFork { + gothrow("stack growth after fork") + } + if thisg.m.morebuf.g != thisg.m.curg { + print("runtime: newstack called from g=", thisg.m.morebuf.g, "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n") + morebuf := thisg.m.morebuf + traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g) + gothrow("runtime: wrong goroutine in newstack") + } + if thisg.m.curg.throwsplit { + gp := thisg.m.curg + // Update syscallsp, syscallpc in case traceback uses them. + morebuf := thisg.m.morebuf + gp.syscallsp = morebuf.sp + gp.syscallpc = morebuf.pc + print("runtime: newstack sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n", + "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n", + "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n") + gothrow("runtime: stack split at bad time") + } + + // The goroutine must be executing in order to call newstack, + // so it must be Grunning or Gscanrunning. + + gp := thisg.m.curg + morebuf := thisg.m.morebuf + thisg.m.morebuf.pc = 0 + thisg.m.morebuf.lr = 0 + thisg.m.morebuf.sp = 0 + thisg.m.morebuf.g = nil + + casgstatus(gp, _Grunning, _Gwaiting) + gp.waitreason = "stack growth" + + rewindmorestack(&gp.sched) + + if gp.stack.lo == 0 { + gothrow("missing stack in newstack") + } + sp := gp.sched.sp + if thechar == '6' || thechar == '8' { + // The call to morestack cost a word. + sp -= ptrSize + } + if stackDebug >= 1 || sp < gp.stack.lo { + print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n", + "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n", + "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n") + } + if sp < gp.stack.lo { + print("runtime: gp=", gp, ", gp->status=", hex(readgstatus(gp)), "\n ") + print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n") + gothrow("runtime: split stack overflow") + } + + if gp.stackguard0 == stackPreempt { + if gp == thisg.m.g0 { + gothrow("runtime: preempt g0") + } + if thisg.m.p == nil && thisg.m.locks == 0 { + gothrow("runtime: g is running but p is not") + } + if gp.preemptscan { + gcphasework(gp) + casgstatus(gp, _Gwaiting, _Grunning) + gp.stackguard0 = gp.stack.lo + _StackGuard + gp.preempt = false + gp.preemptscan = false // Tells the GC premption was successful. + gogo(&gp.sched) // never return + } + + // Be conservative about where we preempt. + // We are interested in preempting user Go code, not runtime code. + if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.gcing != 0 || thisg.m.p.status != _Prunning { + // Let the goroutine keep running for now. + // gp->preempt is set, so it will be preempted next time. + gp.stackguard0 = gp.stack.lo + _StackGuard + casgstatus(gp, _Gwaiting, _Grunning) + gogo(&gp.sched) // never return + } + + // Act like goroutine called runtime.Gosched. + casgstatus(gp, _Gwaiting, _Grunning) + gosched_m(gp) // never return + } + + // Allocate a bigger segment and move the stack. + oldsize := int(gp.stack.hi - gp.stack.lo) + newsize := oldsize * 2 + if uintptr(newsize) > maxstacksize { + print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n") + gothrow("stack overflow") + } + + // Note that the concurrent GC might be scanning the stack as we try to replace it. + // copystack takes care of the appropriate coordination with the stack scanner. + copystack(gp, uintptr(newsize)) + if stackDebug >= 1 { + print("stack grow done\n") + } + casgstatus(gp, _Gwaiting, _Grunning) + gogo(&gp.sched) +} + +//go:nosplit +func nilfunc() { + *(*uint8)(nil) = 0 +} + +// adjust Gobuf as if it executed a call to fn +// and then did an immediate gosave. +func gostartcallfn(gobuf *gobuf, fv *funcval) { + var fn unsafe.Pointer + if fv != nil { + fn = (unsafe.Pointer)(fv.fn) + } else { + fn = unsafe.Pointer(funcPC(nilfunc)) + } + gostartcall(gobuf, fn, (unsafe.Pointer)(fv)) +} + +// Maybe shrink the stack being used by gp. +// Called at garbage collection time. +func shrinkstack(gp *g) { + if readgstatus(gp) == _Gdead { + if gp.stack.lo != 0 { + // Free whole stack - it will get reallocated + // if G is used again. + stackfree(gp.stack) + gp.stack.lo = 0 + gp.stack.hi = 0 + } + return + } + if gp.stack.lo == 0 { + gothrow("missing stack in shrinkstack") + } + + oldsize := gp.stack.hi - gp.stack.lo + newsize := oldsize / 2 + if newsize < _FixedStack { + return // don't shrink below the minimum-sized stack + } + used := gp.stack.hi - gp.sched.sp + if used >= oldsize/4 { + return // still using at least 1/4 of the segment. + } + + // We can't copy the stack if we're in a syscall. + // The syscall might have pointers into the stack. + if gp.syscallsp != 0 { + return + } + + /* TODO + if goos_windows && gp.m != nil && gp.m.libcallsp != 0 { + return + } + */ + + if stackDebug > 0 { + print("shrinking stack ", oldsize, "->", newsize, "\n") + } + copystack(gp, newsize) +} + +// Do any delayed stack freeing that was queued up during GC. +func shrinkfinish() { + lock(&stackpoolmu) + s := stackfreequeue + stackfreequeue = stack{} + unlock(&stackpoolmu) + for s.lo != 0 { + t := *(*stack)(unsafe.Pointer(s.lo)) + stackfree(s) + s = t + } +} + +//go:nosplit +func morestackc() { + systemstack(func() { + gothrow("attempt to execute C code on Go stack") + }) +} diff --git a/src/runtime/stack2.go b/src/runtime/stack2.go new file mode 100644 index 000000000..e50b32c78 --- /dev/null +++ b/src/runtime/stack2.go @@ -0,0 +1,106 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +/* +Stack layout parameters. +Included both by runtime (compiled via 6c) and linkers (compiled via gcc). + +The per-goroutine g->stackguard is set to point StackGuard bytes +above the bottom of the stack. Each function compares its stack +pointer against g->stackguard to check for overflow. To cut one +instruction from the check sequence for functions with tiny frames, +the stack is allowed to protrude StackSmall bytes below the stack +guard. Functions with large frames don't bother with the check and +always call morestack. The sequences are (for amd64, others are +similar): + + guard = g->stackguard + frame = function's stack frame size + argsize = size of function arguments (call + return) + + stack frame size <= StackSmall: + CMPQ guard, SP + JHI 3(PC) + MOVQ m->morearg, $(argsize << 32) + CALL morestack(SB) + + stack frame size > StackSmall but < StackBig + LEAQ (frame-StackSmall)(SP), R0 + CMPQ guard, R0 + JHI 3(PC) + MOVQ m->morearg, $(argsize << 32) + CALL morestack(SB) + + stack frame size >= StackBig: + MOVQ m->morearg, $((argsize << 32) | frame) + CALL morestack(SB) + +The bottom StackGuard - StackSmall bytes are important: there has +to be enough room to execute functions that refuse to check for +stack overflow, either because they need to be adjacent to the +actual caller's frame (deferproc) or because they handle the imminent +stack overflow (morestack). + +For example, deferproc might call malloc, which does one of the +above checks (without allocating a full frame), which might trigger +a call to morestack. This sequence needs to fit in the bottom +section of the stack. On amd64, morestack's frame is 40 bytes, and +deferproc's frame is 56 bytes. That fits well within the +StackGuard - StackSmall bytes at the bottom. +The linkers explore all possible call traces involving non-splitting +functions to make sure that this limit cannot be violated. +*/ + +const ( + // StackSystem is a number of additional bytes to add + // to each stack below the usual guard area for OS-specific + // purposes like signal handling. Used on Windows and on + // Plan 9 because they do not use a separate stack. + _StackSystem = goos_windows*512*ptrSize + goos_plan9*512 + + // The minimum size of stack used by Go code + _StackMin = 2048 + + // The minimum stack size to allocate. + // The hackery here rounds FixedStack0 up to a power of 2. + _FixedStack0 = _StackMin + _StackSystem + _FixedStack1 = _FixedStack0 - 1 + _FixedStack2 = _FixedStack1 | (_FixedStack1 >> 1) + _FixedStack3 = _FixedStack2 | (_FixedStack2 >> 2) + _FixedStack4 = _FixedStack3 | (_FixedStack3 >> 4) + _FixedStack5 = _FixedStack4 | (_FixedStack4 >> 8) + _FixedStack6 = _FixedStack5 | (_FixedStack5 >> 16) + _FixedStack = _FixedStack6 + 1 + + // Functions that need frames bigger than this use an extra + // instruction to do the stack split check, to avoid overflow + // in case SP - framesize wraps below zero. + // This value can be no bigger than the size of the unmapped + // space at zero. + _StackBig = 4096 + + // The stack guard is a pointer this many bytes above the + // bottom of the stack. + _StackGuard = 512 + _StackSystem + + // After a stack split check the SP is allowed to be this + // many bytes below the stack guard. This saves an instruction + // in the checking sequence for tiny frames. + _StackSmall = 128 + + // The maximum number of bytes that a chain of NOSPLIT + // functions can use. + _StackLimit = _StackGuard - _StackSystem - _StackSmall +) + +// Goroutine preemption request. +// Stored into g->stackguard0 to cause split stack check failure. +// Must be greater than any real sp. +// 0xfffffade in hex. +const ( + _StackPreempt = uintptrMask & -1314 + _StackFork = uintptrMask & -1234 +) diff --git a/src/runtime/string.c b/src/runtime/string.c deleted file mode 100644 index ed5debc33..000000000 --- a/src/runtime/string.c +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "race.h" -#include "textflag.h" - -String runtime·emptystring; - -#pragma textflag NOSPLIT -intgo -runtime·findnull(byte *s) -{ - intgo l; - - if(s == nil) - return 0; - for(l=0; s[l]!=0; l++) - ; - return l; -} - -intgo -runtime·findnullw(uint16 *s) -{ - intgo l; - - if(s == nil) - return 0; - for(l=0; s[l]!=0; l++) - ; - return l; -} - -uintptr runtime·maxstring = 256; // a hint for print - -#pragma textflag NOSPLIT -String -runtime·gostringnocopy(byte *str) -{ - String s; - uintptr ms; - - s.str = str; - s.len = runtime·findnull(str); - while(true) { - ms = runtime·maxstring; - if(s.len <= ms || runtime·casp((void**)&runtime·maxstring, (void*)ms, (void*)s.len)) - return s; - } -} - -// TODO: move this elsewhere -enum -{ - Bit1 = 7, - Bitx = 6, - Bit2 = 5, - Bit3 = 4, - Bit4 = 3, - Bit5 = 2, - - Tx = ((1<<(Bitx+1))-1) ^ 0xFF, /* 1000 0000 */ - T2 = ((1<<(Bit2+1))-1) ^ 0xFF, /* 1100 0000 */ - T3 = ((1<<(Bit3+1))-1) ^ 0xFF, /* 1110 0000 */ - T4 = ((1<<(Bit4+1))-1) ^ 0xFF, /* 1111 0000 */ - - Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0111 1111 */ - Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0111 1111 1111 */ - Rune3 = (1<<(Bit3+2*Bitx))-1, /* 1111 1111 1111 1111 */ - - Maskx = (1<<Bitx)-1, /* 0011 1111 */ - - Runeerror = 0xFFFD, - - SurrogateMin = 0xD800, - SurrogateMax = 0xDFFF, - - Runemax = 0x10FFFF, /* maximum rune value */ -}; - -static int32 -runetochar(byte *str, int32 rune) /* note: in original, arg2 was pointer */ -{ - /* Runes are signed, so convert to unsigned for range check. */ - uint32 c; - - /* - * one character sequence - * 00000-0007F => 00-7F - */ - c = rune; - if(c <= Rune1) { - str[0] = c; - return 1; - } - - /* - * two character sequence - * 0080-07FF => T2 Tx - */ - if(c <= Rune2) { - str[0] = T2 | (c >> 1*Bitx); - str[1] = Tx | (c & Maskx); - return 2; - } - - /* - * If the Rune is out of range or a surrogate half, convert it to the error rune. - * Do this test here because the error rune encodes to three bytes. - * Doing it earlier would duplicate work, since an out of range - * Rune wouldn't have fit in one or two bytes. - */ - if (c > Runemax) - c = Runeerror; - if (SurrogateMin <= c && c <= SurrogateMax) - c = Runeerror; - - /* - * three character sequence - * 0800-FFFF => T3 Tx Tx - */ - if (c <= Rune3) { - str[0] = T3 | (c >> 2*Bitx); - str[1] = Tx | ((c >> 1*Bitx) & Maskx); - str[2] = Tx | (c & Maskx); - return 3; - } - - /* - * four character sequence (21-bit value) - * 10000-1FFFFF => T4 Tx Tx Tx - */ - str[0] = T4 | (c >> 3*Bitx); - str[1] = Tx | ((c >> 2*Bitx) & Maskx); - str[2] = Tx | ((c >> 1*Bitx) & Maskx); - str[3] = Tx | (c & Maskx); - return 4; -} - -String runtime·gostringsize(intgo); - -String -runtime·gostringw(uint16 *str) -{ - intgo n1, n2, i; - byte buf[8]; - String s; - - n1 = 0; - for(i=0; str[i]; i++) - n1 += runetochar(buf, str[i]); - s = runtime·gostringsize(n1+4); - n2 = 0; - for(i=0; str[i]; i++) { - // check for race - if(n2 >= n1) - break; - n2 += runetochar(s.str+n2, str[i]); - } - s.len = n2; - s.str[s.len] = 0; - return s; -} - -int32 -runtime·strcmp(byte *s1, byte *s2) -{ - uintptr i; - byte c1, c2; - - for(i=0;; i++) { - c1 = s1[i]; - c2 = s2[i]; - if(c1 < c2) - return -1; - if(c1 > c2) - return +1; - if(c1 == 0) - return 0; - } -} - -int32 -runtime·strncmp(byte *s1, byte *s2, uintptr n) -{ - uintptr i; - byte c1, c2; - - for(i=0; i<n; i++) { - c1 = s1[i]; - c2 = s2[i]; - if(c1 < c2) - return -1; - if(c1 > c2) - return +1; - if(c1 == 0) - break; - } - return 0; -} - -byte* -runtime·strstr(byte *s1, byte *s2) -{ - byte *sp1, *sp2; - - if(*s2 == 0) - return s1; - for(; *s1; s1++) { - if(*s1 != *s2) - continue; - sp1 = s1; - sp2 = s2; - for(;;) { - if(*sp2 == 0) - return s1; - if(*sp1++ != *sp2++) - break; - } - } - return nil; -} diff --git a/src/runtime/string.go b/src/runtime/string.go index 0809f89bc..e01bc3b84 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -39,22 +39,18 @@ func concatstrings(a []string) string { return s } -//go:nosplit func concatstring2(a [2]string) string { return concatstrings(a[:]) } -//go:nosplit func concatstring3(a [3]string) string { return concatstrings(a[:]) } -//go:nosplit func concatstring4(a [4]string) string { return concatstrings(a[:]) } -//go:nosplit func concatstring5(a [5]string) string { return concatstrings(a[:]) } @@ -225,7 +221,7 @@ func rawbyteslice(size int) (b []byte) { // rawruneslice allocates a new rune slice. The rune slice is not zeroed. func rawruneslice(size int) (b []rune) { - if uintptr(size) > maxmem/4 { + if uintptr(size) > _MaxMem/4 { gothrow("out of memory") } mem := goroundupsize(uintptr(size) * 4) @@ -255,9 +251,6 @@ func gostringsize(n int) string { return s } -//go:noescape -func findnull(*byte) int - func gostring(p *byte) string { l := findnull(p) if l == 0 { @@ -296,3 +289,12 @@ func contains(s, t string) bool { func hasprefix(s, t string) bool { return len(s) >= len(t) && s[:len(t)] == t } + +func goatoi(s string) int { + n := 0 + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + n = n*10 + int(s[0]) - '0' + s = s[1:] + } + return n +} diff --git a/src/runtime/string1.go b/src/runtime/string1.go new file mode 100644 index 000000000..35cde43be --- /dev/null +++ b/src/runtime/string1.go @@ -0,0 +1,108 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:nosplit +func findnull(s *byte) int { + if s == nil { + return 0 + } + p := (*[_MaxMem/2 - 1]byte)(unsafe.Pointer(s)) + l := 0 + for p[l] != 0 { + l++ + } + return l +} + +func findnullw(s *uint16) int { + if s == nil { + return 0 + } + p := (*[_MaxMem/2/2 - 1]uint16)(unsafe.Pointer(s)) + l := 0 + for p[l] != 0 { + l++ + } + return l +} + +var maxstring uintptr = 256 // a hint for print + +//go:nosplit +func gostringnocopy(str *byte) string { + var s string + sp := (*stringStruct)(unsafe.Pointer(&s)) + sp.str = unsafe.Pointer(str) + sp.len = findnull(str) + for { + ms := maxstring + if uintptr(len(s)) <= ms || casuintptr(&maxstring, ms, uintptr(len(s))) { + break + } + } + return s +} + +func gostringw(strw *uint16) string { + var buf [8]byte + str := (*[_MaxMem/2/2 - 1]uint16)(unsafe.Pointer(strw)) + n1 := 0 + for i := 0; str[i] != 0; i++ { + n1 += runetochar(buf[:], rune(str[i])) + } + s, b := rawstring(n1 + 4) + n2 := 0 + for i := 0; str[i] != 0; i++ { + // check for race + if n2 >= n1 { + break + } + n2 += runetochar(b[n2:], rune(str[i])) + } + b[n2] = 0 // for luck + return s[:n2] +} + +func strcmp(s1, s2 *byte) int32 { + p1 := (*[_MaxMem/2 - 1]byte)(unsafe.Pointer(s1)) + p2 := (*[_MaxMem/2 - 1]byte)(unsafe.Pointer(s2)) + + for i := uintptr(0); ; i++ { + c1 := p1[i] + c2 := p2[i] + if c1 < c2 { + return -1 + } + if c1 > c2 { + return +1 + } + if c1 == 0 { + return 0 + } + } +} + +func strncmp(s1, s2 *byte, n uintptr) int32 { + p1 := (*[_MaxMem/2 - 1]byte)(unsafe.Pointer(s1)) + p2 := (*[_MaxMem/2 - 1]byte)(unsafe.Pointer(s2)) + + for i := uintptr(0); i < n; i++ { + c1 := p1[i] + c2 := p2[i] + if c1 < c2 { + return -1 + } + if c1 > c2 { + return +1 + } + if c1 == 0 { + break + } + } + return 0 +} diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 1282397ad..aa7577cf9 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -23,12 +23,7 @@ func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer { return unsafe.Pointer(uintptr(p) + delta) } -// in runtime.c func getg() *g -func acquirem() *m -func releasem(mp *m) -func gomcache() *mcache -func readgstatus(*g) uint32 // proc.c // mcall switches from the g to the g0 stack and invokes fn(g), // where g is the goroutine that made the call. @@ -43,85 +38,30 @@ func readgstatus(*g) uint32 // proc.c //go:noescape func mcall(fn func(*g)) -// onM switches from the g to the g0 stack and invokes fn(). -// When fn returns, onM switches back to the g and returns, -// continuing execution on the g stack. -// If arguments must be passed to fn, they can be written to -// g->m->ptrarg (pointers) and g->m->scalararg (non-pointers) -// before the call and then consulted during fn. -// Similarly, fn can pass return values back in those locations. -// If fn is written in Go, it can be a closure, which avoids the need for -// ptrarg and scalararg entirely. -// After reading values out of ptrarg and scalararg it is conventional -// to zero them to avoid (memory or information) leaks. +// systemstack runs fn on a system stack. +// If systemstack is called from the per-OS-thread (g0) stack, or +// if systemstack is called from the signal handling (gsignal) stack, +// systemstack calls fn directly and returns. +// Otherwise, systemstack is being called from the limited stack +// of an ordinary goroutine. In this case, systemstack switches +// to the per-OS-thread stack, calls fn, and switches back. +// It is common to use a func literal as the argument, in order +// to share inputs and outputs with the code around the call +// to system stack: // -// If onM is called from a g0 stack, it invokes fn and returns, -// without any stack switches. -// -// If onM is called from a gsignal stack, it crashes the program. -// The implication is that functions used in signal handlers must -// not use onM. -// -// NOTE(rsc): We could introduce a separate onMsignal that is -// like onM but if called from a gsignal stack would just run fn on -// that stack. The caller of onMsignal would be required to save the -// old values of ptrarg/scalararg and restore them when the call -// was finished, in case the signal interrupted an onM sequence -// in progress on the g or g0 stacks. Until there is a clear need for this, -// we just reject onM in signal handling contexts entirely. -// -//go:noescape -func onM(fn func()) - -// onMsignal is like onM but is allowed to be used in code that -// might run on the gsignal stack. Code running on a signal stack -// may be interrupting an onM sequence on the main stack, so -// if the onMsignal calling sequence writes to ptrarg/scalararg, -// it must first save the old values and then restore them when -// finished. As an exception to the rule, it is fine not to save and -// restore the values if the program is trying to crash rather than -// return from the signal handler. -// Once all the runtime is written in Go, there will be no ptrarg/scalararg -// and the distinction between onM and onMsignal (and perhaps mcall) -// can go away. -// -// If onMsignal is called from a gsignal stack, it invokes fn directly, -// without a stack switch. Otherwise onMsignal behaves like onM. +// ... set up y ... +// systemstack(func() { +// x = bigcall(y) +// }) +// ... use x ... // //go:noescape -func onM_signalok(fn func()) +func systemstack(fn func()) -func badonm() { - gothrow("onM called from signal goroutine") +func badsystemstack() { + gothrow("systemstack called from unexpected goroutine") } -// C functions that run on the M stack. -// Call using mcall. -func gosched_m(*g) -func park_m(*g) -func recovery_m(*g) - -// More C functions that run on the M stack. -// Call using onM. -func mcacheRefill_m() -func largeAlloc_m() -func gc_m() -func scavenge_m() -func setFinalizer_m() -func removeFinalizer_m() -func markallocated_m() -func unrollgcprog_m() -func unrollgcproginplace_m() -func setgcpercent_m() -func setmaxthreads_m() -func ready_m() -func deferproc_m() -func goexit_m() -func startpanic_m() -func dopanic_m() -func readmemstats_m() -func writeheapdump_m() - // memclr clears n bytes starting at ptr. // in memclr_*.s //go:noescape @@ -132,12 +72,6 @@ func memclr(ptr unsafe.Pointer, n uintptr) //go:noescape func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) -func starttheworld() -func stoptheworld() -func newextram() -func lockOSThread() -func unlockOSThread() - // exported value for testing var hashLoad = loadFactor @@ -159,16 +93,9 @@ func noescape(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(x ^ 0) } -func entersyscall() -func reentersyscall(pc uintptr, sp unsafe.Pointer) -func entersyscallblock() -func exitsyscall() - func cgocallback(fn, frame unsafe.Pointer, framesize uintptr) func gogo(buf *gobuf) func gosave(buf *gobuf) -func read(fd int32, p unsafe.Pointer, n int32) int32 -func close(fd int32) int32 func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 //go:noescape @@ -176,35 +103,31 @@ func jmpdefer(fv *funcval, argp uintptr) func exit1(code int32) func asminit() func setg(gg *g) -func exit(code int32) func breakpoint() -func nanotime() int64 -func usleep(usec uint32) - -// careful: cputicks is not guaranteed to be monotonic! In particular, we have -// noticed drift between cpus on certain os/arch combinations. See issue 8976. -func cputicks() int64 -func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer -func munmap(addr unsafe.Pointer, n uintptr) -func madvise(addr unsafe.Pointer, n uintptr, flags int32) func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32) -func osyield() func procyield(cycles uint32) func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr) -func readgogc() int32 -func purgecachedstats(c *mcache) -func gostringnocopy(b *byte) string func goexit() //go:noescape -func write(fd uintptr, p unsafe.Pointer, n int32) int32 - -//go:noescape func cas(ptr *uint32, old, new uint32) bool -//go:noescape -func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool +// casp cannot have a go:noescape annotation, because +// while ptr and old do not escape, new does. If new is marked as +// not escaping, the compiler will make incorrect escape analysis +// decisions about the value being xchg'ed. +// Instead, make casp a wrapper around the actual atomic. +// When calling the wrapper we mark ptr as noescape explicitly. + +//go:nosplit +func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { + return casp1((*unsafe.Pointer)(noescape(unsafe.Pointer(ptr))), noescape(old), new) +} + +func casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool + +func nop() // call to prevent inlining of function body //go:noescape func casuintptr(ptr *uintptr, old, new uintptr) bool @@ -261,18 +184,10 @@ func asmcgocall(fn, arg unsafe.Pointer) //go:noescape func asmcgocall_errno(fn, arg unsafe.Pointer) int32 -//go:noescape -func open(name *byte, mode, perm int32) int32 - -//go:noescape -func gotraceback(*bool) int32 - +// argp used in Defer structs when there is no argp. const _NoArgs = ^uintptr(0) -func newstack() -func newproc() func morestack() -func mstart() func rt0_go() // return0 is a stub used to return 0 from deferproc. @@ -314,3 +229,5 @@ func call134217728(fn, arg unsafe.Pointer, n, retoffset uint32) func call268435456(fn, arg unsafe.Pointer, n, retoffset uint32) func call536870912(fn, arg unsafe.Pointer, n, retoffset uint32) func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32) + +func systemstack_switch() diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go new file mode 100644 index 000000000..cb0b0f0ed --- /dev/null +++ b/src/runtime/stubs2.go @@ -0,0 +1,28 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !solaris +// +build !windows + +package runtime + +import "unsafe" + +func read(fd int32, p unsafe.Pointer, n int32) int32 +func close(fd int32) int32 + +func exit(code int32) +func nanotime() int64 +func usleep(usec uint32) + +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer +func munmap(addr unsafe.Pointer, n uintptr) + +//go:noescape +func write(fd uintptr, p unsafe.Pointer, n int32) int32 + +//go:noescape +func open(name *byte, mode, perm int32) int32 + +func madvise(addr unsafe.Pointer, n uintptr, flags int32) diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 45d107b77..749a289cd 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -22,8 +22,7 @@ func (f *Func) raw() *_func { // funcdata.h const ( - _PCDATA_ArgSize = 0 - _PCDATA_StackMapIndex = 1 + _PCDATA_StackMapIndex = 0 _FUNCDATA_ArgsPointerMaps = 0 _FUNCDATA_LocalsPointerMaps = 1 _FUNCDATA_DeadValueMaps = 2 @@ -122,8 +121,8 @@ func (f *Func) Entry() uintptr { func (f *Func) FileLine(pc uintptr) (file string, line int) { // Pass strict=false here, because anyone can call this function, // and they might just be wrong about targetpc belonging to f. - line = int(funcline1(f.raw(), pc, &file, false)) - return file, line + file, line32 := funcline1(f.raw(), pc, false) + return file, int(line32) } func findfunc(pc uintptr) *_func { @@ -208,20 +207,19 @@ func gofuncname(f *_func) string { return gostringnocopy(funcname(f)) } -func funcline1(f *_func, targetpc uintptr, file *string, strict bool) int32 { - *file = "?" +func funcline1(f *_func, targetpc uintptr, strict bool) (file string, line int32) { fileno := int(pcvalue(f, f.pcfile, targetpc, strict)) - line := pcvalue(f, f.pcln, targetpc, strict) + line = pcvalue(f, f.pcln, targetpc, strict) if fileno == -1 || line == -1 || fileno >= len(filetab) { // print("looking for ", hex(targetpc), " in ", gofuncname(f), " got file=", fileno, " line=", lineno, "\n") - return 0 + return "?", 0 } - *file = gostringnocopy(&pclntable[filetab[fileno]]) - return line + file = gostringnocopy(&pclntable[filetab[fileno]]) + return } -func funcline(f *_func, targetpc uintptr, file *string) int32 { - return funcline1(f, targetpc, file, true) +func funcline(f *_func, targetpc uintptr) (file string, line int32) { + return funcline1(f, targetpc, true) } func funcspdelta(f *_func, targetpc uintptr) int32 { diff --git a/src/runtime/sys_arm.c b/src/runtime/sys_arm.c deleted file mode 100644 index a65560e5b..000000000 --- a/src/runtime/sys_arm.c +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" - -// adjust Gobuf as if it executed a call to fn with context ctxt -// and then did an immediate Gosave. -void -runtime·gostartcall(Gobuf *gobuf, void (*fn)(void), void *ctxt) -{ - if(gobuf->lr != 0) - runtime·throw("invalid use of gostartcall"); - gobuf->lr = gobuf->pc; - gobuf->pc = (uintptr)fn; - gobuf->ctxt = ctxt; -} - -// Called to rewind context saved during morestack back to beginning of function. -// To help us, the linker emits a jmp back to the beginning right after the -// call to morestack. We just have to decode and apply that jump. -void -runtime·rewindmorestack(Gobuf *gobuf) -{ - uint32 inst; - - inst = *(uint32*)gobuf->pc; - if((gobuf->pc&3) == 0 && (inst>>24) == 0x9a) { - //runtime·printf("runtime: rewind pc=%p to pc=%p\n", gobuf->pc, gobuf->pc + ((int32)(inst<<8)>>6) + 8); - gobuf->pc += ((int32)(inst<<8)>>6) + 8; - return; - } - runtime·printf("runtime: pc=%p %x\n", gobuf->pc, inst); - runtime·throw("runtime: misuse of rewindmorestack"); -} diff --git a/src/runtime/sys_arm.go b/src/runtime/sys_arm.go new file mode 100644 index 000000000..81777c710 --- /dev/null +++ b/src/runtime/sys_arm.go @@ -0,0 +1,35 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// adjust Gobuf as if it executed a call to fn with context ctxt +// and then did an immediate Gosave. +func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { + if buf.lr != 0 { + gothrow("invalid use of gostartcall") + } + buf.lr = buf.pc + buf.pc = uintptr(fn) + buf.ctxt = ctxt +} + +// Called to rewind context saved during morestack back to beginning of function. +// To help us, the linker emits a jmp back to the beginning right after the +// call to morestack. We just have to decode and apply that jump. +func rewindmorestack(buf *gobuf) { + var inst uint32 + if buf.pc&3 == 0 && buf.pc != 0 { + inst = *(*uint32)(unsafe.Pointer(buf.pc)) + if inst>>24 == 0x9a { + buf.pc += uintptr(int32(inst<<8)>>6) + 8 + return + } + } + + print("runtime: pc=", hex(buf.pc), " ", hex(inst), "\n") + gothrow("runtime: misuse of rewindmorestack") +} diff --git a/src/runtime/sys_darwin_386.s b/src/runtime/sys_darwin_386.s index a961c71a8..7cb5695e7 100644 --- a/src/runtime/sys_darwin_386.s +++ b/src/runtime/sys_darwin_386.s @@ -6,7 +6,8 @@ // See http://fxr.watson.org/fxr/source/bsd/kern/syscalls.c?v=xnu-1228 // or /usr/include/sys/syscall.h (on a Mac) for system call numbers. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // Exit the entire program (like C exit) @@ -248,7 +249,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$40 MOVL BX, 0(SP) MOVL $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVL DI, 20(SP) @@ -275,7 +276,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$40 MOVL 20(SP), DI MOVL DI, g(CX) -sigtramp_ret: +ret: // call sigreturn MOVL context+16(FP), CX MOVL style+4(FP), BX diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index bd397d72a..0a955f982 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -11,7 +11,8 @@ // The high 8 bits specify the kind of system call: 1=Mach, 2=BSD, 3=Machine-Dependent. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // Exit the entire program (like C exit) @@ -211,7 +212,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$64 MOVL DX, 0(SP) MOVQ $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVQ R10, 48(SP) @@ -233,7 +234,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$64 MOVQ 48(SP), R10 MOVQ R10, g(BX) -sigtramp_ret: +ret: // call sigreturn MOVL $(0x2000000+184), AX // sigreturn(ucontext, infostyle) MOVQ 32(SP), DI // saved ucontext diff --git a/src/runtime/sys_dragonfly_386.s b/src/runtime/sys_dragonfly_386.s index 161eaec19..bb4903ef6 100644 --- a/src/runtime/sys_dragonfly_386.s +++ b/src/runtime/sys_dragonfly_386.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT runtime·sys_umtx_sleep(SB),NOSPLIT,$-4 @@ -217,7 +218,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL BX, 0(SP) MOVL $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVL DI, 20(SP) @@ -243,7 +244,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL 20(SP), BX MOVL BX, g(CX) -sigtramp_ret: +ret: // call sigreturn MOVL context+8(FP), AX MOVL $0, 0(SP) // syscall gap diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index 2c756018c..db07ed703 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT runtime·sys_umtx_sleep(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index 2c40fc433..d1f67c3fc 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT runtime·sys_umtx_op(SB),NOSPLIT,$-4 @@ -197,7 +198,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL BX, 0(SP) MOVL $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVL DI, 20(SP) @@ -223,7 +224,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL 20(SP), BX MOVL BX, g(CX) -sigtramp_ret: +ret: // call sigreturn MOVL context+8(FP), AX MOVL $0, 0(SP) // syscall gap diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index 65f8c1a6e..84cee51d8 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // FreeBSD 8, FreeBSD 9, and older versions that I have checked diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s index d875138b6..198b427bf 100644 --- a/src/runtime/sys_freebsd_arm.s +++ b/src/runtime/sys_freebsd_arm.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // for EABI, as we don't support OABI @@ -362,7 +363,7 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0 SWI $0 RET -TEXT runtime·casp(SB),NOSPLIT,$0 +TEXT runtime·casp1(SB),NOSPLIT,$0 B runtime·cas(SB) // TODO(minux): this is only valid for ARMv6+ diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 0f6d4bbb5..1861f237f 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -6,7 +6,8 @@ // System calls and other sys.stuff for 386, Linux // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT runtime·exit(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 33b91e872..6d4dfdbd2 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -6,7 +6,8 @@ // System calls and other sys.stuff for AMD64, Linux // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT runtime·exit(SB),NOSPLIT,$0-4 @@ -115,7 +116,7 @@ TEXT time·now(SB),NOSPLIT,$16 // That leaves 104 for the gettime code to use. Hope that's enough! MOVQ runtime·__vdso_clock_gettime_sym(SB), AX CMPQ AX, $0 - JEQ fallback_gtod + JEQ fallback MOVL $0, DI // CLOCK_REALTIME LEAQ 0(SP), SI CALL AX @@ -124,7 +125,7 @@ TEXT time·now(SB),NOSPLIT,$16 MOVQ AX, sec+0(FP) MOVL DX, nsec+8(FP) RET -fallback_gtod: +fallback: LEAQ 0(SP), DI MOVQ $0, SI MOVQ runtime·__vdso_gettimeofday_sym(SB), AX @@ -141,7 +142,7 @@ TEXT runtime·nanotime(SB),NOSPLIT,$16 // See comment above in time.now. MOVQ runtime·__vdso_clock_gettime_sym(SB), AX CMPQ AX, $0 - JEQ fallback_gtod_nt + JEQ fallback MOVL $1, DI // CLOCK_MONOTONIC LEAQ 0(SP), SI CALL AX @@ -153,7 +154,7 @@ TEXT runtime·nanotime(SB),NOSPLIT,$16 ADDQ DX, AX MOVQ AX, ret+0(FP) RET -fallback_gtod_nt: +fallback: LEAQ 0(SP), DI MOVQ $0, SI MOVQ runtime·__vdso_gettimeofday_sym(SB), AX diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index bd285f399..21d97fda9 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -6,7 +6,8 @@ // System calls and other sys.stuff for arm, Linux // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // for EABI, as we don't support OABI @@ -373,25 +374,25 @@ TEXT cas<>(SB),NOSPLIT,$0 TEXT runtime·cas(SB),NOSPLIT,$0 MOVW ptr+0(FP), R2 MOVW old+4(FP), R0 -casagain: +loop: MOVW new+8(FP), R1 BL cas<>(SB) - BCC cascheck + BCC check MOVW $1, R0 MOVB R0, ret+12(FP) RET -cascheck: +check: // Kernel lies; double-check. MOVW ptr+0(FP), R2 MOVW old+4(FP), R0 MOVW 0(R2), R3 CMP R0, R3 - BEQ casagain + BEQ loop MOVW $0, R0 MOVB R0, ret+12(FP) RET -TEXT runtime·casp(SB),NOSPLIT,$0 +TEXT runtime·casp1(SB),NOSPLIT,$0 B runtime·cas(SB) TEXT runtime·osyield(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_linux_power64x.s b/src/runtime/sys_linux_power64x.s new file mode 100644 index 000000000..395f657bf --- /dev/null +++ b/src/runtime/sys_linux_power64x.s @@ -0,0 +1,384 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux +// +build power64 power64le + +// +// System calls and other sys.stuff for Power64, Linux +// + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" + +#define SYS_exit 1 +#define SYS_read 3 +#define SYS_write 4 +#define SYS_open 5 +#define SYS_close 6 +#define SYS_fcntl 55 +#define SYS_gettimeofday 78 +#define SYS_select 82 // always return -ENOSYS +#define SYS_mmap 90 +#define SYS_munmap 91 +#define SYS_setitimer 104 +#define SYS_clone 120 +#define SYS_newselect 142 +#define SYS_sched_yield 158 +#define SYS_rt_sigreturn 172 +#define SYS_rt_sigaction 173 +#define SYS_rt_sigprocmask 174 +#define SYS_sigaltstack 185 +#define SYS_ugetrlimit 190 +#define SYS_madvise 205 +#define SYS_mincore 206 +#define SYS_gettid 207 +#define SYS_tkill 208 +#define SYS_futex 221 +#define SYS_sched_getaffinity 223 +#define SYS_exit_group 234 +#define SYS_epoll_create 236 +#define SYS_epoll_ctl 237 +#define SYS_epoll_wait 238 +#define SYS_clock_gettime 246 +#define SYS_epoll_create1 315 + +TEXT runtime·exit(SB),NOSPLIT,$-8-4 + MOVW code+0(FP), R3 + SYSCALL $SYS_exit_group + RETURN + +TEXT runtime·exit1(SB),NOSPLIT,$-8-4 + MOVW code+0(FP), R3 + SYSCALL $SYS_exit + RETURN + +TEXT runtime·open(SB),NOSPLIT,$-8-20 + MOVD name+0(FP), R3 + MOVW mode+8(FP), R4 + MOVW perm+12(FP), R5 + SYSCALL $SYS_open + MOVW R3, ret+16(FP) + RETURN + +TEXT runtime·close(SB),NOSPLIT,$-8-12 + MOVW fd+0(FP), R3 + SYSCALL $SYS_close + MOVW R3, ret+8(FP) + RETURN + +TEXT runtime·write(SB),NOSPLIT,$-8-28 + MOVD fd+0(FP), R3 + MOVD p+8(FP), R4 + MOVW n+16(FP), R5 + SYSCALL $SYS_write + MOVW R3, ret+24(FP) + RETURN + +TEXT runtime·read(SB),NOSPLIT,$-8-28 + MOVW fd+0(FP), R3 + MOVD p+8(FP), R4 + MOVW n+16(FP), R5 + SYSCALL $SYS_read + MOVW R3, ret+24(FP) + RETURN + +TEXT runtime·getrlimit(SB),NOSPLIT,$-8-20 + MOVW kind+0(FP), R3 + MOVD limit+8(FP), R4 + SYSCALL $SYS_ugetrlimit + MOVW R3, ret+16(FP) + RETURN + +TEXT runtime·usleep(SB),NOSPLIT,$16-4 + MOVW usec+0(FP), R3 + MOVD R3, R5 + MOVW $1000000, R4 + DIVD R4, R3 + MOVD R3, 8(R1) + MULLD R3, R4 + SUB R4, R5 + MOVD R5, 16(R1) + + // select(0, 0, 0, 0, &tv) + MOVW $0, R3 + MOVW $0, R4 + MOVW $0, R5 + MOVW $0, R6 + ADD $8, R1, R7 + SYSCALL $SYS_newselect + RETURN + +TEXT runtime·raise(SB),NOSPLIT,$-8 + SYSCALL $SYS_gettid + MOVW R3, R3 // arg 1 tid + MOVW sig+0(FP), R4 // arg 2 + SYSCALL $SYS_tkill + RETURN + +TEXT runtime·setitimer(SB),NOSPLIT,$-8-24 + MOVW mode+0(FP), R3 + MOVD new+8(FP), R4 + MOVD old+16(FP), R5 + SYSCALL $SYS_setitimer + RETURN + +TEXT runtime·mincore(SB),NOSPLIT,$-8-28 + MOVD addr+0(FP), R3 + MOVD n+8(FP), R4 + MOVD dst+16(FP), R5 + SYSCALL $SYS_mincore + MOVW R3, ret+24(FP) + RETURN + +// func now() (sec int64, nsec int32) +TEXT time·now(SB),NOSPLIT,$16 + MOVD $0(R1), R3 + MOVD $0, R4 + SYSCALL $SYS_gettimeofday + MOVD 0(R1), R3 // sec + MOVD 8(R1), R5 // usec + MOVD $1000, R4 + MULLD R4, R5 + MOVD R3, sec+0(FP) + MOVW R5, nsec+8(FP) + RETURN + +TEXT runtime·nanotime(SB),NOSPLIT,$16 + MOVW $1, R3 // CLOCK_MONOTONIC + MOVD $0(R1), R4 + SYSCALL $SYS_clock_gettime + MOVD 0(R1), R3 // sec + MOVD 8(R1), R5 // nsec + // sec is in R3, nsec in R5 + // return nsec in R3 + MOVD $1000000000, R4 + MULLD R4, R3 + ADD R5, R3 + MOVD R3, ret+0(FP) + RETURN + +TEXT runtime·rtsigprocmask(SB),NOSPLIT,$-8-28 + MOVW sig+0(FP), R3 + MOVD new+8(FP), R4 + MOVD old+16(FP), R5 + MOVW size+24(FP), R6 + SYSCALL $SYS_rt_sigprocmask + BVC 2(PC) + MOVD R0, 0xf1(R0) // crash + RETURN + +TEXT runtime·rt_sigaction(SB),NOSPLIT,$-8-36 + MOVD sig+0(FP), R3 + MOVD new+8(FP), R4 + MOVD old+16(FP), R5 + MOVD size+24(FP), R6 + SYSCALL $SYS_rt_sigaction + MOVW R3, ret+32(FP) + RETURN + +#ifdef GOARCH_power64le +// power64le doesn't need function descriptors +TEXT runtime·sigtramp(SB),NOSPLIT,$64 +#else +// function descriptor for the real sigtramp +TEXT runtime·sigtramp(SB),NOSPLIT,$-8 + DWORD $runtime·_sigtramp(SB) + DWORD $0 + DWORD $0 +TEXT runtime·_sigtramp(SB),NOSPLIT,$64 +#endif + // initialize essential registers (just in case) + BL runtime·reginit(SB) + + // check that g exists + CMP g, $0 + BNE 6(PC) + MOVD R3, 8(R1) + MOVD $runtime·badsignal(SB), R31 + MOVD R31, CTR + BL (CTR) + RETURN + + // save g + MOVD g, 40(R1) + MOVD g, R6 + + // g = m->gsignal + MOVD g_m(g), R7 + MOVD m_gsignal(R7), g + + MOVW R3, 8(R1) + MOVD R4, 16(R1) + MOVD R5, 24(R1) + MOVD R6, 32(R1) + + BL runtime·sighandler(SB) + + // restore g + MOVD 40(R1), g + + RETURN + +TEXT runtime·mmap(SB),NOSPLIT,$-8 + MOVD addr+0(FP), R3 + MOVD n+8(FP), R4 + MOVW prot+16(FP), R5 + MOVW flags+20(FP), R6 + MOVW fd+24(FP), R7 + MOVW off+28(FP), R8 + + SYSCALL $SYS_mmap + MOVD R3, ret+32(FP) + RETURN + +TEXT runtime·munmap(SB),NOSPLIT,$-8 + MOVD addr+0(FP), R3 + MOVD n+8(FP), R4 + SYSCALL $SYS_munmap + BVC 2(PC) + MOVD R0, 0xf3(R0) + RETURN + +TEXT runtime·madvise(SB),NOSPLIT,$-8 + MOVD addr+0(FP), R3 + MOVD n+8(FP), R4 + MOVW flags+16(FP), R5 + SYSCALL $SYS_madvise + // ignore failure - maybe pages are locked + RETURN + +// int64 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex(SB),NOSPLIT,$-8 + MOVD addr+0(FP), R3 + MOVW op+8(FP), R4 + MOVW val+12(FP), R5 + MOVD ts+16(FP), R6 + MOVD addr2+24(FP), R7 + MOVW val3+32(FP), R8 + SYSCALL $SYS_futex + MOVW R3, ret+40(FP) + RETURN + +// int64 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void)); +TEXT runtime·clone(SB),NOSPLIT,$-8 + MOVW flags+0(FP), R3 + MOVD stk+8(FP), R4 + + // Copy mp, gp, fn off parent stack for use by child. + // Careful: Linux system call clobbers ???. + MOVD mm+16(FP), R7 + MOVD gg+24(FP), R8 + MOVD fn+32(FP), R12 + + MOVD R7, -8(R4) + MOVD R8, -16(R4) + MOVD R12, -24(R4) + MOVD $1234, R7 + MOVD R7, -32(R4) + + SYSCALL $SYS_clone + + // In parent, return. + CMP R3, $0 + BEQ 3(PC) + MOVW R3, ret+40(FP) + RETURN + + // In child, on new stack. + // initialize essential registers + BL runtime·reginit(SB) + MOVD -32(R1), R7 + CMP R7, $1234 + BEQ 2(PC) + MOVD R0, 0(R0) + + // Initialize m->procid to Linux tid + SYSCALL $SYS_gettid + + MOVD -24(R1), R12 + MOVD -16(R1), R8 + MOVD -8(R1), R7 + + MOVD R3, m_procid(R7) + + // TODO: setup TLS. + + // In child, set up new stack + MOVD R7, g_m(R8) + MOVD R8, g + //CALL runtime·stackcheck(SB) + + // Call fn + MOVD R12, CTR + BL (CTR) + + // It shouldn't return. If it does, exit + MOVW $111, R3 + SYSCALL $SYS_exit_group + BR -2(PC) // keep exiting + +TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 + MOVD new+0(FP), R3 + MOVD old+8(FP), R4 + SYSCALL $SYS_sigaltstack + BVC 2(PC) + MOVD R0, 0xf1(R0) // crash + RETURN + +TEXT runtime·osyield(SB),NOSPLIT,$-8 + SYSCALL $SYS_sched_yield + RETURN + +TEXT runtime·sched_getaffinity(SB),NOSPLIT,$-8 + MOVD pid+0(FP), R3 + MOVD len+8(FP), R4 + MOVD buf+16(FP), R5 + SYSCALL $SYS_sched_getaffinity + MOVW R3, ret+24(FP) + RETURN + +// int32 runtime·epollcreate(int32 size); +TEXT runtime·epollcreate(SB),NOSPLIT,$-8 + MOVW size+0(FP), R3 + SYSCALL $SYS_epoll_create + MOVW R3, ret+8(FP) + RETURN + +// int32 runtime·epollcreate1(int32 flags); +TEXT runtime·epollcreate1(SB),NOSPLIT,$-8 + MOVW flags+0(FP), R3 + SYSCALL $SYS_epoll_create1 + MOVW R3, ret+8(FP) + RETURN + +// func epollctl(epfd, op, fd int32, ev *epollEvent) int +TEXT runtime·epollctl(SB),NOSPLIT,$-8 + MOVW epfd+0(FP), R3 + MOVW op+4(FP), R4 + MOVW fd+8(FP), R5 + MOVD ev+16(FP), R6 + SYSCALL $SYS_epoll_ctl + MOVW R3, ret+24(FP) + RETURN + +// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); +TEXT runtime·epollwait(SB),NOSPLIT,$-8 + MOVW epfd+0(FP), R3 + MOVD ev+8(FP), R4 + MOVW nev+16(FP), R5 + MOVW timeout+20(FP), R6 + SYSCALL $SYS_epoll_wait + MOVW R3, ret+24(FP) + RETURN + +// void runtime·closeonexec(int32 fd); +TEXT runtime·closeonexec(SB),NOSPLIT,$-8 + MOVW fd+0(FP), R3 // fd + MOVD $2, R4 // F_SETFD + MOVD $1, R5 // FD_CLOEXEC + SYSCALL $SYS_fcntl + RETURN diff --git a/src/runtime/sys_nacl_386.s b/src/runtime/sys_nacl_386.s index 47985f31f..85c8175b1 100644 --- a/src/runtime/sys_nacl_386.s +++ b/src/runtime/sys_nacl_386.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" #include "syscall_nacl.h" @@ -293,7 +294,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0 MOVL $0, 0(SP) MOVL $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVL DI, 20(SP) @@ -317,7 +318,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0 MOVL 20(SP), BX MOVL BX, g(CX) -sigtramp_ret: +ret: // Enable exceptions again. NACL_SYSCALL(SYS_exception_clear_flag) diff --git a/src/runtime/sys_nacl_amd64p32.s b/src/runtime/sys_nacl_amd64p32.s index 4eb4aacdd..f5624ca8d 100644 --- a/src/runtime/sys_nacl_amd64p32.s +++ b/src/runtime/sys_nacl_amd64p32.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" #include "syscall_nacl.h" @@ -338,7 +339,6 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$80 MOVL 20(SP), BX MOVL BX, g(CX) -sigtramp_ret: // Enable exceptions again. NACL_SYSCALL(SYS_exception_clear_flag) diff --git a/src/runtime/sys_nacl_arm.s b/src/runtime/sys_nacl_arm.s index d354ab483..ded95a86b 100644 --- a/src/runtime/sys_nacl_arm.s +++ b/src/runtime/sys_nacl_arm.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" #include "syscall_nacl.h" @@ -269,7 +270,6 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$80 // restore g MOVW 20(R13), g -sigtramp_ret: // Enable exceptions again. NACL_SYSCALL(SYS_exception_clear_flag) @@ -301,7 +301,7 @@ nog: TEXT runtime·nacl_sysinfo(SB),NOSPLIT,$16 RET -TEXT runtime·casp(SB),NOSPLIT,$0 +TEXT runtime·casp1(SB),NOSPLIT,$0 B runtime·cas(SB) // This is only valid for ARMv6+, however, NaCl/ARM is only defined diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 23f2f6bd1..509d6d4a8 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // Exit the entire program (like C exit) diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index eb9766d3f..e26d60667 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // int32 lwp_create(void *context, uintptr flags, void *lwpid) diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s index 039a0832e..fa9bc577a 100644 --- a/src/runtime/sys_netbsd_arm.s +++ b/src/runtime/sys_netbsd_arm.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // Exit the entire program (like C exit) @@ -330,7 +331,7 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0 SWI $0xa0005c // sys_fcntl RET -TEXT runtime·casp(SB),NOSPLIT,$0 +TEXT runtime·casp1(SB),NOSPLIT,$0 B runtime·cas(SB) // TODO(minux): this is only valid for ARMv6+ diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s index 5cda7768a..93907577e 100644 --- a/src/runtime/sys_openbsd_386.s +++ b/src/runtime/sys_openbsd_386.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" #define CLOCK_MONOTONIC $3 @@ -186,7 +187,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL BX, 0(SP) MOVL $runtime·badsignal(SB), AX CALL AX - JMP sigtramp_ret + JMP ret // save g MOVL DI, 20(SP) @@ -212,7 +213,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$44 MOVL 20(SP), BX MOVL BX, g(CX) -sigtramp_ret: +ret: // call sigreturn MOVL context+8(FP), AX MOVL $0, 0(SP) // syscall gap diff --git a/src/runtime/sys_openbsd_amd64.s b/src/runtime/sys_openbsd_amd64.s index 4e9db2390..9dc0fb685 100644 --- a/src/runtime/sys_openbsd_amd64.s +++ b/src/runtime/sys_openbsd_amd64.s @@ -6,7 +6,8 @@ // /usr/src/sys/kern/syscalls.master for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" #define CLOCK_MONOTONIC $3 diff --git a/src/runtime/sys_plan9_386.s b/src/runtime/sys_plan9_386.s index a41b56258..b9db8cbf1 100644 --- a/src/runtime/sys_plan9_386.s +++ b/src/runtime/sys_plan9_386.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // setldt(int entry, int address, int limit) diff --git a/src/runtime/sys_plan9_amd64.s b/src/runtime/sys_plan9_amd64.s index 3a96c2bf9..02c7c8743 100644 --- a/src/runtime/sys_plan9_amd64.s +++ b/src/runtime/sys_plan9_amd64.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // setldt(int entry, int address, int limit) diff --git a/src/runtime/sys_power64x.go b/src/runtime/sys_power64x.go new file mode 100644 index 000000000..90ebde7b4 --- /dev/null +++ b/src/runtime/sys_power64x.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build power64 power64le + +package runtime + +import "unsafe" + +// adjust Gobuf as if it executed a call to fn with context ctxt +// and then did an immediate Gosave. +func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { + if buf.lr != 0 { + gothrow("invalid use of gostartcall") + } + buf.lr = buf.pc + buf.pc = uintptr(fn) + buf.ctxt = ctxt +} + +// Called to rewind context saved during morestack back to beginning of function. +// To help us, the linker emits a jmp back to the beginning right after the +// call to morestack. We just have to decode and apply that jump. +func rewindmorestack(buf *gobuf) { + var inst uint32 + if buf.pc&3 == 0 && buf.pc != 0 { + inst = *(*uint32)(unsafe.Pointer(buf.pc)) + if inst>>26 == 18 && inst&3 == 0 { + //print("runtime: rewind pc=", hex(buf.pc), " to pc=", hex(uintptr(buf.pc + int32(inst<<6)>>6)), "\n"); + buf.pc += uintptr(int32(inst<<6) >> 6) + return + } + } + print("runtime: pc=", hex(buf.pc), " ", hex(inst), "\n") + gothrow("runtime: misuse of rewindmorestack") +} diff --git a/src/runtime/sys_solaris_amd64.s b/src/runtime/sys_solaris_amd64.s index 0ebdab6ee..54aeaeaf5 100644 --- a/src/runtime/sys_solaris_amd64.s +++ b/src/runtime/sys_solaris_amd64.s @@ -6,14 +6,15 @@ // /usr/include/sys/syscall.h for syscall numbers. // -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // This is needed by asm_amd64.s TEXT runtime·settls(SB),NOSPLIT,$8 RET -// void libc·miniterrno(void *(*___errno)(void)); +// void libc_miniterrno(void *(*___errno)(void)); // // Set the TLS errno pointer in M. // @@ -40,7 +41,7 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$0 SUBQ $64, SP // 16 bytes will do, but who knows in the future? MOVQ $3, DI // CLOCK_REALTIME from <sys/time_impl.h> MOVQ SP, SI - MOVQ libc·clock_gettime(SB), AX + MOVQ libc_clock_gettime(SB), AX CALL AX MOVQ (SP), AX // tv_sec from struct timespec IMULQ $1000000000, AX // multiply into nanoseconds @@ -53,7 +54,7 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$0 TEXT runtime·pipe1(SB),NOSPLIT,$0 SUBQ $16, SP // 8 bytes will do, but stack has to be 16-byte alligned MOVQ SP, DI - MOVQ libc·pipe(SB), AX + MOVQ libc_pipe(SB), AX CALL AX MOVL 0(SP), AX MOVL 4(SP), DX @@ -132,7 +133,7 @@ TEXT runtime·tstart_sysvicall(SB),NOSPLIT,$0 MOVQ AX, (g_stack+stack_hi)(DX) SUBQ $(0x100000), AX // stack size MOVQ AX, (g_stack+stack_lo)(DX) - ADDQ $const_StackGuard, AX + ADDQ $const__StackGuard, AX MOVQ AX, g_stackguard0(DX) MOVQ AX, g_stackguard1(DX) @@ -287,24 +288,24 @@ TEXT runtime·usleep1(SB),NOSPLIT,$0 // Execute call on m->g0. get_tls(R15) CMPQ R15, $0 - JE usleep1_noswitch + JE noswitch MOVQ g(R15), R13 CMPQ R13, $0 - JE usleep1_noswitch + JE noswitch MOVQ g_m(R13), R13 CMPQ R13, $0 - JE usleep1_noswitch + JE noswitch // TODO(aram): do something about the cpu profiler here. MOVQ m_g0(R13), R14 CMPQ g(R15), R14 - JNE usleep1_switch + JNE switch // executing on m->g0 already CALL AX RET -usleep1_switch: +switch: // Switch to m->g0 stack and back. MOVQ (g_sched+gobuf_sp)(R14), R14 MOVQ SP, -8(R14) @@ -313,20 +314,20 @@ usleep1_switch: MOVQ 0(SP), SP RET -usleep1_noswitch: +noswitch: // Not a Go-managed thread. Do not switch stack. CALL AX RET // Runs on OS stack. duration (in µs units) is in DI. TEXT runtime·usleep2(SB),NOSPLIT,$0 - MOVQ libc·usleep(SB), AX + MOVQ libc_usleep(SB), AX CALL AX RET // Runs on OS stack, called from runtime·osyield. TEXT runtime·osyield1(SB),NOSPLIT,$0 - MOVQ libc·sched_yield(SB), AX + MOVQ libc_sched_yield(SB), AX CALL AX RET diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index 932fe9dd2..c8a830cdf 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // void runtime·asmstdcall(void *c); @@ -43,7 +44,7 @@ TEXT runtime·badsignal2(SB),NOSPLIT,$24 // stderr MOVL $-12, 0(SP) MOVL SP, BP - CALL *runtime·GetStdHandle(SB) + CALL *runtime·_GetStdHandle(SB) MOVL BP, SP MOVL AX, 0(SP) // handle @@ -55,7 +56,7 @@ TEXT runtime·badsignal2(SB),NOSPLIT,$24 MOVL $0, 0(DX) MOVL DX, 12(SP) MOVL $0, 16(SP) // overlapped - CALL *runtime·WriteFile(SB) + CALL *runtime·_WriteFile(SB) MOVL BP, SI RET @@ -106,7 +107,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0 MOVL g_m(DX), BX MOVL m_g0(BX), BX CMPL DX, BX - JEQ sigtramp_g0 + JEQ g0 // switch to the g0 stack get_tls(BP) @@ -123,7 +124,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0 MOVL SP, 36(DI) MOVL DI, SP -sigtramp_g0: +g0: MOVL 0(CX), BX // ExceptionRecord* MOVL 4(CX), CX // Context* MOVL BX, 0(SP) @@ -207,7 +208,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0 MOVL BX, g_m(SP) LEAL -8192(SP), CX MOVL CX, (g_stack+stack_lo)(SP) - ADDL $const_StackGuard, CX + ADDL $const__StackGuard, CX MOVL CX, g_stackguard0(SP) MOVL CX, g_stackguard1(SP) MOVL DX, (g_stack+stack_hi)(SP) @@ -254,8 +255,8 @@ TEXT runtime·callbackasm1+0(SB),NOSPLIT,$0 MOVL -4(BX)(AX*4), BX // extract callback context - MOVL cbctxt_gobody(BX), AX - MOVL cbctxt_argsize(BX), DX + MOVL wincallbackcontext_gobody(BX), AX + MOVL wincallbackcontext_argsize(BX), DX // preserve whatever's at the memory location that // the callback will use to store the return value @@ -265,7 +266,7 @@ TEXT runtime·callbackasm1+0(SB),NOSPLIT,$0 ADDL $4, DX // remember how to restore stack on return - MOVL cbctxt_restorestack(BX), BX + MOVL wincallbackcontext_restorestack(BX), BX PUSHL BX // call target Go function @@ -313,7 +314,7 @@ TEXT runtime·tstart(SB),NOSPLIT,$0 MOVL AX, (g_stack+stack_hi)(DX) SUBL $(64*1024), AX // stack size MOVL AX, (g_stack+stack_lo)(DX) - ADDL $const_StackGuard, AX + ADDL $const__StackGuard, AX MOVL AX, g_stackguard0(DX) MOVL AX, g_stackguard1(DX) @@ -383,12 +384,12 @@ TEXT runtime·usleep1(SB),NOSPLIT,$0 MOVL m_g0(BP), SI CMPL g(CX), SI - JNE usleep1_switch + JNE switch // executing on m->g0 already CALL AX - JMP usleep1_ret + JMP ret -usleep1_switch: +switch: // Switch to m->g0 stack and back. MOVL (g_sched+gobuf_sp)(SI), SI MOVL SP, -4(SI) @@ -396,7 +397,7 @@ usleep1_switch: CALL AX MOVL 0(SP), SP -usleep1_ret: +ret: get_tls(CX) MOVL g(CX), BP MOVL g_m(BP), BP @@ -414,7 +415,7 @@ TEXT runtime·usleep2(SB),NOSPLIT,$20 MOVL $0, alertable-16(SP) MOVL $-1, handle-20(SP) MOVL SP, BP - MOVL runtime·NtWaitForSingleObject(SB), AX + MOVL runtime·_NtWaitForSingleObject(SB), AX CALL AX MOVL BP, SP RET diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index e6190ce68..68f7cd392 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" // maxargs should be divisible by 2, as Windows stack @@ -65,7 +66,7 @@ TEXT runtime·badsignal2(SB),NOSPLIT,$48 // stderr MOVQ $-12, CX // stderr MOVQ CX, 0(SP) - MOVQ runtime·GetStdHandle(SB), AX + MOVQ runtime·_GetStdHandle(SB), AX CALL AX MOVQ AX, CX // handle @@ -78,7 +79,7 @@ TEXT runtime·badsignal2(SB),NOSPLIT,$48 MOVQ $0, 0(R9) MOVQ R9, 24(SP) MOVQ $0, 32(SP) // overlapped - MOVQ runtime·WriteFile(SB), AX + MOVQ runtime·_WriteFile(SB), AX CALL AX RET @@ -138,7 +139,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0 MOVQ g_m(DX), BX MOVQ m_g0(BX), BX CMPQ DX, BX - JEQ sigtramp_g0 + JEQ g0 // switch to g0 stack get_tls(BP) @@ -157,7 +158,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0 MOVQ SP, 104(DI) MOVQ DI, SP -sigtramp_g0: +g0: MOVQ 0(CX), BX // ExceptionRecord* MOVQ 8(CX), CX // Context* MOVQ BX, 0(SP) @@ -244,7 +245,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0 LEAQ -8192(SP), CX MOVQ CX, (g_stack+stack_lo)(SP) - ADDQ $const_StackGuard, CX + ADDQ $const__StackGuard, CX MOVQ CX, g_stackguard0(SP) MOVQ CX, g_stackguard1(SP) MOVQ DX, (g_stack+stack_hi)(SP) @@ -293,8 +294,8 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVQ -8(CX)(AX*8), AX // extract callback context - MOVQ cbctxt_argsize(AX), DX - MOVQ cbctxt_gobody(AX), AX + MOVQ wincallbackcontext_argsize(AX), DX + MOVQ wincallbackcontext_gobody(AX), AX // preserve whatever's at the memory location that // the callback will use to store the return value @@ -354,7 +355,7 @@ TEXT runtime·tstart_stdcall(SB),NOSPLIT,$0 MOVQ AX, (g_stack+stack_hi)(DX) SUBQ $(64*1024), AX // stack size MOVQ AX, (g_stack+stack_lo)(DX) - ADDQ $const_StackGuard, AX + ADDQ $const__StackGuard, AX MOVQ AX, g_stackguard0(DX) MOVQ AX, g_stackguard1(DX) @@ -407,12 +408,12 @@ TEXT runtime·usleep1(SB),NOSPLIT,$0 MOVQ m_g0(R13), R14 CMPQ g(R15), R14 - JNE usleep1_switch + JNE switch // executing on m->g0 already CALL AX - JMP usleep1_ret + JMP ret -usleep1_switch: +switch: // Switch to m->g0 stack and back. MOVQ (g_sched+gobuf_sp)(R14), R14 MOVQ SP, -8(R14) @@ -420,7 +421,7 @@ usleep1_switch: CALL AX MOVQ 0(SP), SP -usleep1_ret: +ret: MOVQ $0, m_libcallsp(R13) RET @@ -435,7 +436,7 @@ TEXT runtime·usleep2(SB),NOSPLIT,$16 MOVQ BX, (R8) MOVQ $-1, CX // handle MOVQ $0, DX // alertable - MOVQ runtime·NtWaitForSingleObject(SB), AX + MOVQ runtime·_NtWaitForSingleObject(SB), AX CALL AX MOVQ 8(SP), SP RET diff --git a/src/runtime/sys_x86.c b/src/runtime/sys_x86.go index a450b3e58..086af8ff1 100644 --- a/src/runtime/sys_x86.c +++ b/src/runtime/sys_x86.go @@ -4,54 +4,51 @@ // +build amd64 amd64p32 386 -#include "runtime.h" +package runtime + +import "unsafe" // adjust Gobuf as it if executed a call to fn with context ctxt // and then did an immediate gosave. -void -runtime·gostartcall(Gobuf *gobuf, void (*fn)(void), void *ctxt) -{ - uintptr *sp; - - sp = (uintptr*)gobuf->sp; - if(sizeof(uintreg) > sizeof(uintptr)) - *--sp = 0; - *--sp = (uintptr)gobuf->pc; - gobuf->sp = (uintptr)sp; - gobuf->pc = (uintptr)fn; - gobuf->ctxt = ctxt; +func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { + sp := buf.sp + if regSize > ptrSize { + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(sp)) = 0 + } + sp -= ptrSize + *(*uintptr)(unsafe.Pointer(sp)) = buf.pc + buf.sp = sp + buf.pc = uintptr(fn) + buf.ctxt = ctxt } // Called to rewind context saved during morestack back to beginning of function. // To help us, the linker emits a jmp back to the beginning right after the // call to morestack. We just have to decode and apply that jump. -void -runtime·rewindmorestack(Gobuf *gobuf) -{ - byte *pc; - - pc = (byte*)gobuf->pc; - if(pc[0] == 0xe9) { // jmp 4-byte offset - gobuf->pc = gobuf->pc + 5 + *(int32*)(pc+1); - return; +func rewindmorestack(buf *gobuf) { + pc := (*[8]byte)(unsafe.Pointer(buf.pc)) + if pc[0] == 0xe9 { // jmp 4-byte offset + buf.pc = buf.pc + 5 + uintptr(int64(*(*int32)(unsafe.Pointer(&pc[1])))) + return } - if(pc[0] == 0xeb) { // jmp 1-byte offset - gobuf->pc = gobuf->pc + 2 + *(int8*)(pc+1); - return; + if pc[0] == 0xeb { // jmp 1-byte offset + buf.pc = buf.pc + 2 + uintptr(int64(*(*int8)(unsafe.Pointer(&pc[1])))) + return } - if(pc[0] == 0xcc) { + if pc[0] == 0xcc { // This is a breakpoint inserted by gdb. We could use // runtime·findfunc to find the function. But if we // do that, then we will continue execution at the // function entry point, and we will not hit the gdb // breakpoint. So for this case we don't change - // gobuf->pc, so that when we return we will execute + // buf.pc, so that when we return we will execute // the jump instruction and carry on. This means that // stack unwinding may not work entirely correctly // (http://golang.org/issue/5723) but the user is // running under gdb anyhow. - return; + return } - runtime·printf("runtime: pc=%p %x %x %x %x %x\n", pc, pc[0], pc[1], pc[2], pc[3], pc[4]); - runtime·throw("runtime: misuse of rewindmorestack"); + print("runtime: pc=", pc, " ", hex(pc[0]), " ", hex(pc[1]), " ", hex(pc[2]), " ", hex(pc[3]), " ", hex(pc[4]), "\n") + gothrow("runtime: misuse of rewindmorestack") } diff --git a/src/runtime/syscall2_solaris.go b/src/runtime/syscall2_solaris.go new file mode 100644 index 000000000..f4ffa7410 --- /dev/null +++ b/src/runtime/syscall2_solaris.go @@ -0,0 +1,47 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import _ "unsafe" + +//go:cgo_import_dynamic libc_chdir chdir "libc.so" +//go:cgo_import_dynamic libc_chroot chroot "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_dlclose dlclose "libc.so" +//go:cgo_import_dynamic libc_dlopen dlopen "libc.so" +//go:cgo_import_dynamic libc_dlsym dlsym "libc.so" +//go:cgo_import_dynamic libc_execve execve "libc.so" +//go:cgo_import_dynamic libc_fcntl fcntl "libc.so" +//go:cgo_import_dynamic libc_gethostname gethostname "libc.so" +//go:cgo_import_dynamic libc_ioctl ioctl "libc.so" +//go:cgo_import_dynamic libc_pipe pipe "libc.so" +//go:cgo_import_dynamic libc_setgid setgid "libc.so" +//go:cgo_import_dynamic libc_setgroups setgroups "libc.so" +//go:cgo_import_dynamic libc_setsid setsid "libc.so" +//go:cgo_import_dynamic libc_setuid setuid "libc.so" +//go:cgo_import_dynamic libc_setpgid setsid "libc.so" +//go:cgo_import_dynamic libc_syscall syscall "libc.so" +//go:cgo_import_dynamic libc_forkx forkx "libc.so" +//go:cgo_import_dynamic libc_wait4 wait4 "libc.so" + +//go:linkname libc_chdir libc_chdir +//go:linkname libc_chroot libc_chroot +//go:linkname libc_close libc_close +//go:linkname libc_dlclose libc_dlclose +//go:linkname libc_dlopen libc_dlopen +//go:linkname libc_dlsym libc_dlsym +//go:linkname libc_execve libc_execve +//go:linkname libc_fcntl libc_fcntl +//go:linkname libc_gethostname libc_gethostname +//go:linkname libc_ioctl libc_ioctl +//go:linkname libc_pipe libc_pipe +//go:linkname libc_setgid libc_setgid +//go:linkname libc_setgroups libc_setgroups +//go:linkname libc_setsid libc_setsid +//go:linkname libc_setuid libc_setuid +//go:linkname libc_setpgid libc_setpgid +//go:linkname libc_syscall libc_syscall +//go:linkname libc_forkx libc_forkx +//go:linkname libc_wait4 libc_wait4 diff --git a/src/runtime/syscall_solaris.c b/src/runtime/syscall_solaris.c deleted file mode 100644 index 13ac31bde..000000000 --- a/src/runtime/syscall_solaris.c +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#pragma dynimport libc·chdir chdir "libc.so" -#pragma dynimport libc·chroot chroot "libc.so" -#pragma dynimport libc·close close "libc.so" -#pragma dynimport libc·dlclose dlclose "libc.so" -#pragma dynimport libc·dlopen dlopen "libc.so" -#pragma dynimport libc·dlsym dlsym "libc.so" -#pragma dynimport libc·execve execve "libc.so" -#pragma dynimport libc·fcntl fcntl "libc.so" -#pragma dynimport libc·gethostname gethostname "libc.so" -#pragma dynimport libc·ioctl ioctl "libc.so" -#pragma dynimport libc·pipe pipe "libc.so" -#pragma dynimport libc·setgid setgid "libc.so" -#pragma dynimport libc·setgroups setgroups "libc.so" -#pragma dynimport libc·setsid setsid "libc.so" -#pragma dynimport libc·setuid setuid "libc.so" -#pragma dynimport libc·setpgid setsid "libc.so" -#pragma dynimport libc·syscall syscall "libc.so" -#pragma dynimport libc·forkx forkx "libc.so" -#pragma dynimport libc·wait4 wait4 "libc.so" diff --git a/src/runtime/syscall_solaris.go b/src/runtime/syscall_solaris.go index 50d3a1d36..9b9971674 100644 --- a/src/runtime/syscall_solaris.go +++ b/src/runtime/syscall_solaris.go @@ -9,12 +9,10 @@ import "unsafe" var ( libc_chdir, libc_chroot, - libc_close, libc_dlopen, libc_dlclose, libc_dlsym, libc_execve, - libc_exit, libc_fcntl, libc_forkx, libc_gethostname, @@ -27,7 +25,6 @@ var ( libc_setpgid, libc_syscall, libc_wait4, - libc_write, pipe1 libcFunc ) @@ -38,9 +35,9 @@ func syscall_sysvicall6(fn, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err n: nargs, args: uintptr(unsafe.Pointer(&a1)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) return call.r1, call.r2, call.err } @@ -62,7 +59,7 @@ func syscall_rawsysvicall6(fn, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, e //go:nosplit func syscall_chdir(path uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_chdir)), + fn: uintptr(unsafe.Pointer(libc_chdir)), n: 1, args: uintptr(unsafe.Pointer(&path)), } @@ -73,7 +70,7 @@ func syscall_chdir(path uintptr) (err uintptr) { //go:nosplit func syscall_chroot(path uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_chroot)), + fn: uintptr(unsafe.Pointer(libc_chroot)), n: 1, args: uintptr(unsafe.Pointer(&path)), } @@ -84,18 +81,18 @@ func syscall_chroot(path uintptr) (err uintptr) { // like close, but must not split stack, for forkx. //go:nosplit func syscall_close(fd int32) int32 { - return int32(sysvicall1(&libc_close, uintptr(fd))) + return int32(sysvicall1(libc_close, uintptr(fd))) } func syscall_dlopen(name *byte, mode uintptr) (handle uintptr, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_dlopen)), + fn: uintptr(unsafe.Pointer(libc_dlopen)), n: 2, args: uintptr(unsafe.Pointer(&name)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) if call.r1 == 0 { return call.r1, call.err } @@ -104,25 +101,25 @@ func syscall_dlopen(name *byte, mode uintptr) (handle uintptr, err uintptr) { func syscall_dlclose(handle uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_dlclose)), + fn: uintptr(unsafe.Pointer(libc_dlclose)), n: 1, args: uintptr(unsafe.Pointer(&handle)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) return call.r1 } func syscall_dlsym(handle uintptr, name *byte) (proc uintptr, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_dlsym)), + fn: uintptr(unsafe.Pointer(libc_dlsym)), n: 2, args: uintptr(unsafe.Pointer(&handle)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) if call.r1 == 0 { return call.r1, call.err } @@ -132,7 +129,7 @@ func syscall_dlsym(handle uintptr, name *byte) (proc uintptr, err uintptr) { //go:nosplit func syscall_execve(path, argv, envp uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_execve)), + fn: uintptr(unsafe.Pointer(libc_execve)), n: 3, args: uintptr(unsafe.Pointer(&path)), } @@ -143,13 +140,13 @@ func syscall_execve(path, argv, envp uintptr) (err uintptr) { // like exit, but must not split stack, for forkx. //go:nosplit func syscall_exit(code uintptr) { - sysvicall1(&libc_exit, code) + sysvicall1(libc_exit, code) } //go:nosplit func syscall_fcntl(fd, cmd, arg uintptr) (val, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_fcntl)), + fn: uintptr(unsafe.Pointer(libc_fcntl)), n: 3, args: uintptr(unsafe.Pointer(&fd)), } @@ -160,7 +157,7 @@ func syscall_fcntl(fd, cmd, arg uintptr) (val, err uintptr) { //go:nosplit func syscall_forkx(flags uintptr) (pid uintptr, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_forkx)), + fn: uintptr(unsafe.Pointer(libc_forkx)), n: 1, args: uintptr(unsafe.Pointer(&flags)), } @@ -172,13 +169,13 @@ func syscall_gethostname() (name string, err uintptr) { cname := new([_MAXHOSTNAMELEN]byte) var args = [2]uintptr{uintptr(unsafe.Pointer(&cname[0])), _MAXHOSTNAMELEN} call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_gethostname)), + fn: uintptr(unsafe.Pointer(libc_gethostname)), n: 2, args: uintptr(unsafe.Pointer(&args[0])), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) if call.r1 != 0 { return "", call.err } @@ -189,7 +186,7 @@ func syscall_gethostname() (name string, err uintptr) { //go:nosplit func syscall_ioctl(fd, req, arg uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_ioctl)), + fn: uintptr(unsafe.Pointer(libc_ioctl)), n: 3, args: uintptr(unsafe.Pointer(&fd)), } @@ -203,9 +200,9 @@ func syscall_pipe() (r, w, err uintptr) { n: 0, args: uintptr(unsafe.Pointer(&pipe1)), // it's unused but must be non-nil, otherwise crashes } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) return call.r1, call.r2, call.err } @@ -217,7 +214,7 @@ func syscall_pipe() (r, w, err uintptr) { // TODO(aram): make this panic once we stop calling fcntl(2) in net using it. func syscall_rawsyscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_syscall)), + fn: uintptr(unsafe.Pointer(libc_syscall)), n: 4, args: uintptr(unsafe.Pointer(&trap)), } @@ -228,7 +225,7 @@ func syscall_rawsyscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr) { //go:nosplit func syscall_setgid(gid uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_setgid)), + fn: uintptr(unsafe.Pointer(libc_setgid)), n: 1, args: uintptr(unsafe.Pointer(&gid)), } @@ -239,7 +236,7 @@ func syscall_setgid(gid uintptr) (err uintptr) { //go:nosplit func syscall_setgroups(ngid, gid uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_setgroups)), + fn: uintptr(unsafe.Pointer(libc_setgroups)), n: 2, args: uintptr(unsafe.Pointer(&ngid)), } @@ -250,9 +247,9 @@ func syscall_setgroups(ngid, gid uintptr) (err uintptr) { //go:nosplit func syscall_setsid() (pid, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_setsid)), + fn: uintptr(unsafe.Pointer(libc_setsid)), n: 0, - args: uintptr(unsafe.Pointer(&libc_setsid)), // it's unused but must be non-nil, otherwise crashes + args: uintptr(unsafe.Pointer(libc_setsid)), // it's unused but must be non-nil, otherwise crashes } asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) return call.r1, call.err @@ -261,7 +258,7 @@ func syscall_setsid() (pid, err uintptr) { //go:nosplit func syscall_setuid(uid uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_setuid)), + fn: uintptr(unsafe.Pointer(libc_setuid)), n: 1, args: uintptr(unsafe.Pointer(&uid)), } @@ -272,7 +269,7 @@ func syscall_setuid(uid uintptr) (err uintptr) { //go:nosplit func syscall_setpgid(pid, pgid uintptr) (err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_setpgid)), + fn: uintptr(unsafe.Pointer(libc_setpgid)), n: 2, args: uintptr(unsafe.Pointer(&pid)), } @@ -288,32 +285,32 @@ func syscall_setpgid(pid, pgid uintptr) (err uintptr) { // TODO(aram): make this panic once we stop calling fcntl(2) in net using it. func syscall_syscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_syscall)), + fn: uintptr(unsafe.Pointer(libc_syscall)), n: 4, args: uintptr(unsafe.Pointer(&trap)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) return call.r1, call.r2, call.err } func syscall_wait4(pid uintptr, wstatus *uint32, options uintptr, rusage unsafe.Pointer) (wpid int, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_wait4)), + fn: uintptr(unsafe.Pointer(libc_wait4)), n: 4, args: uintptr(unsafe.Pointer(&pid)), } - entersyscallblock() + entersyscallblock(0) asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&call)) - exitsyscall() + exitsyscall(0) return int(call.r1), call.err } //go:nosplit func syscall_write(fd, buf, nbyte uintptr) (n, err uintptr) { call := libcall{ - fn: uintptr(unsafe.Pointer(&libc_write)), + fn: uintptr(unsafe.Pointer(libc_write)), n: 3, args: uintptr(unsafe.Pointer(&fd)), } diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index efbcab510..661ee59d7 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -41,20 +41,20 @@ func callbackasmAddr(i int) uintptr { func compileCallback(fn eface, cleanstack bool) (code uintptr) { if fn._type == nil || (fn._type.kind&kindMask) != kindFunc { - panic("compilecallback: not a function") + panic("compileCallback: not a function") } ft := (*functype)(unsafe.Pointer(fn._type)) - if len(ft.out) != 1 { - panic("compilecallback: function must have one output parameter") + if ft.out.len != 1 { + panic("compileCallback: function must have one output parameter") } uintptrSize := unsafe.Sizeof(uintptr(0)) - if t := (**_type)(unsafe.Pointer(&ft.out[0])); (*t).size != uintptrSize { - panic("compilecallback: output parameter size is wrong") + if t := (**_type)(unsafe.Pointer(ft.out.array)); (*t).size != uintptrSize { + panic("compileCallback: output parameter size is wrong") } argsize := uintptr(0) - for _, t := range (*[1024](*_type))(unsafe.Pointer(&ft.in[0]))[:len(ft.in)] { + for _, t := range (*[1024](*_type))(unsafe.Pointer(ft.in.array))[:ft.in.len] { if (*t).size > uintptrSize { - panic("compilecallback: input parameter size is wrong") + panic("compileCallback: input parameter size is wrong") } argsize += uintptrSize } @@ -87,8 +87,6 @@ func compileCallback(fn eface, cleanstack bool) (code uintptr) { return callbackasmAddr(n) } -func getLoadLibrary() uintptr - //go:nosplit func syscall_loadlibrary(filename *uint16) (handle, err uintptr) { var c libcall @@ -103,8 +101,6 @@ func syscall_loadlibrary(filename *uint16) (handle, err uintptr) { return } -func getGetProcAddress() uintptr - //go:nosplit func syscall_getprocaddress(handle uintptr, procname *byte) (outhandle, err uintptr) { var c libcall diff --git a/src/runtime/thunk.s b/src/runtime/thunk.s index 0a0f147c4..241dd90af 100644 --- a/src/runtime/thunk.s +++ b/src/runtime/thunk.s @@ -4,12 +4,17 @@ // This file exposes various internal runtime functions to other packages in std lib. -#include "zasm_GOOS_GOARCH.h" #include "textflag.h" #ifdef GOARCH_arm #define JMP B #endif +#ifdef GOARCH_power64 +#define JMP BR +#endif +#ifdef GOARCH_power64le +#define JMP BR +#endif TEXT net·runtimeNano(SB),NOSPLIT,$0-0 JMP runtime·nanotime(SB) @@ -181,3 +186,18 @@ TEXT syscall·runtime_envs(SB),NOSPLIT,$0-0 TEXT os·runtime_args(SB),NOSPLIT,$0-0 JMP runtime·runtime_args(SB) + +TEXT sync·runtime_procUnpin(SB),NOSPLIT,$0-0 + JMP runtime·sync_procUnpin(SB) + +TEXT sync·runtime_procPin(SB),NOSPLIT,$0-0 + JMP runtime·sync_procPin(SB) + +TEXT syscall·runtime_BeforeFork(SB),NOSPLIT,$0-0 + JMP runtime·syscall_BeforeFork(SB) + +TEXT syscall·runtime_AfterFork(SB),NOSPLIT,$0-0 + JMP runtime·syscall_AfterFork(SB) + +TEXT reflect·typelinks(SB),NOSPLIT,$0-0 + JMP runtime·typelinks(SB) diff --git a/src/runtime/thunk_solaris_amd64.s b/src/runtime/thunk_solaris_amd64.s deleted file mode 100644 index f61188c14..000000000 --- a/src/runtime/thunk_solaris_amd64.s +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file exposes various external library functions to Go code in the runtime. - -#include "zasm_GOOS_GOARCH.h" -#include "textflag.h" - -TEXT runtime·libc_chdir(SB),NOSPLIT,$0 - MOVQ libc·chdir(SB), AX - JMP AX - -TEXT runtime·libc_chroot(SB),NOSPLIT,$0 - MOVQ libc·chroot(SB), AX - JMP AX - -TEXT runtime·libc_close(SB),NOSPLIT,$0 - MOVQ libc·close(SB), AX - JMP AX - -TEXT runtime·libc_dlopen(SB),NOSPLIT,$0 - MOVQ libc·dlopen(SB), AX - JMP AX - -TEXT runtime·libc_dlclose(SB),NOSPLIT,$0 - MOVQ libc·dlclose(SB), AX - JMP AX - -TEXT runtime·libc_dlsym(SB),NOSPLIT,$0 - MOVQ libc·dlsym(SB), AX - JMP AX - -TEXT runtime·libc_execve(SB),NOSPLIT,$0 - MOVQ libc·execve(SB), AX - JMP AX - -TEXT runtime·libc_exit(SB),NOSPLIT,$0 - MOVQ libc·exit(SB), AX - JMP AX - -TEXT runtime·libc_fcntl(SB),NOSPLIT,$0 - MOVQ libc·fcntl(SB), AX - JMP AX - -TEXT runtime·libc_forkx(SB),NOSPLIT,$0 - MOVQ libc·forkx(SB), AX - JMP AX - -TEXT runtime·libc_gethostname(SB),NOSPLIT,$0 - MOVQ libc·gethostname(SB), AX - JMP AX - -TEXT runtime·libc_ioctl(SB),NOSPLIT,$0 - MOVQ libc·ioctl(SB), AX - JMP AX - -TEXT runtime·libc_setgid(SB),NOSPLIT,$0 - MOVQ libc·setgid(SB), AX - JMP AX - -TEXT runtime·libc_setgroups(SB),NOSPLIT,$0 - MOVQ libc·setgroups(SB), AX - JMP AX - -TEXT runtime·libc_setsid(SB),NOSPLIT,$0 - MOVQ libc·setsid(SB), AX - JMP AX - -TEXT runtime·libc_setuid(SB),NOSPLIT,$0 - MOVQ libc·setuid(SB), AX - JMP AX - -TEXT runtime·libc_setpgid(SB),NOSPLIT,$0 - MOVQ libc·setpgid(SB), AX - JMP AX - -TEXT runtime·libc_syscall(SB),NOSPLIT,$0 - MOVQ libc·syscall(SB), AX - JMP AX - -TEXT runtime·libc_wait4(SB),NOSPLIT,$0 - MOVQ libc·wait4(SB), AX - JMP AX - -TEXT runtime·libc_write(SB),NOSPLIT,$0 - MOVQ libc·write(SB), AX - JMP AX diff --git a/src/runtime/thunk_windows.s b/src/runtime/thunk_windows.s index 7ccb98fd4..b1d5d5753 100644 --- a/src/runtime/thunk_windows.s +++ b/src/runtime/thunk_windows.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" TEXT syscall·Syscall(SB),NOSPLIT,$0-0 diff --git a/src/runtime/tls_arm.s b/src/runtime/tls_arm.s index 85c3940bf..7c5c0e215 100644 --- a/src/runtime/tls_arm.s +++ b/src/runtime/tls_arm.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 1c6ce6e64..e1cc9123f 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -32,15 +32,16 @@ const usesLR = GOARCH != "amd64" && GOARCH != "amd64p32" && GOARCH != "386" var ( // initialized in tracebackinit - deferprocPC uintptr - goexitPC uintptr - jmpdeferPC uintptr - mcallPC uintptr - morestackPC uintptr - mstartPC uintptr - newprocPC uintptr - rt0_goPC uintptr - sigpanicPC uintptr + deferprocPC uintptr + goexitPC uintptr + jmpdeferPC uintptr + mcallPC uintptr + morestackPC uintptr + mstartPC uintptr + newprocPC uintptr + rt0_goPC uintptr + sigpanicPC uintptr + systemstack_switchPC uintptr externalthreadhandlerp uintptr // initialized elsewhere ) @@ -59,6 +60,7 @@ func tracebackinit() { newprocPC = funcPC(newproc) rt0_goPC = funcPC(rt0_go) sigpanicPC = funcPC(sigpanic) + systemstack_switchPC = funcPC(systemstack_switch) } // Traceback over the deferred function calls. @@ -335,8 +337,7 @@ func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf print(hex(argp[i])) } print(")\n") - var file string - line := funcline(f, tracepc, &file) + file, line := funcline(f, tracepc) print("\t", file, ":", line) if frame.pc > f.entry { print(" +", hex(frame.pc-f.entry)) @@ -480,8 +481,7 @@ func printcreatedby(gp *g) { if pc > f.entry { tracepc -= _PCQuantum } - var file string - line := funcline(f, tracepc, &file) + file, line := funcline(f, tracepc) print("\t", file, ":", line) if pc > f.entry { print(" +", hex(pc-f.entry)) @@ -528,7 +528,7 @@ func callers(skip int, pcbuf *uintptr, m int) int { sp := getcallersp(unsafe.Pointer(&skip)) pc := uintptr(getcallerpc(unsafe.Pointer(&skip))) var n int - onM(func() { + systemstack(func() { n = gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) }) return n diff --git a/src/runtime/type.go b/src/runtime/type.go new file mode 100644 index 000000000..cbd5c9ebc --- /dev/null +++ b/src/runtime/type.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Runtime _type representation. + +package runtime + +import "unsafe" + +// Needs to be in sync with ../../cmd/ld/decodesym.c:/^commonsize and pkg/reflect/type.go:/type. +type _type struct { + size uintptr + hash uint32 + _unused uint8 + align uint8 + fieldalign uint8 + kind uint8 + alg unsafe.Pointer + // gc stores _type info required for garbage collector. + // If (kind&KindGCProg)==0, then gc[0] points at sparse GC bitmap + // (no indirection), 4 bits per word. + // If (kind&KindGCProg)!=0, then gc[1] points to a compiler-generated + // read-only GC program; and gc[0] points to BSS space for sparse GC bitmap. + // For huge _types (>MaxGCMask), runtime unrolls the program directly into + // GC bitmap and gc[0] is not used. For moderately-sized _types, runtime + // unrolls the program into gc[0] space on first use. The first byte of gc[0] + // (gc[0][0]) contains 'unroll' flag saying whether the program is already + // unrolled into gc[0] or not. + gc [2]uintptr + _string *string + x *uncommontype + ptrto *_type + zero *byte // ptr to the zero value for this _type +} + +type method struct { + name *string + pkgpath *string + mtyp *_type + typ *_type + ifn unsafe.Pointer + tfn unsafe.Pointer +} + +type uncommontype struct { + name *string + pkgpath *string + mhdr []method + m [0]method +} + +type imethod struct { + name *string + pkgpath *string + _type *_type +} + +type interfacetype struct { + typ _type + mhdr []imethod + m [0]imethod +} + +type maptype struct { + typ _type + key *_type + elem *_type + bucket *_type // internal _type representing a hash bucket + hmap *_type // internal _type representing a hmap + keysize uint8 // size of key slot + indirectkey bool // store ptr to key instead of key itself + valuesize uint8 // size of value slot + indirectvalue bool // store ptr to value instead of value itself + bucketsize uint16 // size of bucket +} + +type chantype struct { + typ _type + elem *_type + dir uintptr +} + +type slicetype struct { + typ _type + elem *_type +} + +type functype struct { + typ _type + dotdotdot bool + in slice + out slice +} + +type ptrtype struct { + typ _type + elem *_type +} diff --git a/src/runtime/type.h b/src/runtime/type.h deleted file mode 100644 index f5b4f9d13..000000000 --- a/src/runtime/type.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Runtime type representation. - -typedef struct Type Type; -typedef struct UncommonType UncommonType; -typedef struct InterfaceType InterfaceType; -typedef struct Method Method; -typedef struct IMethod IMethod; -typedef struct SliceType SliceType; -typedef struct FuncType FuncType; - -// Needs to be in sync with ../../cmd/ld/decodesym.c:/^commonsize and pkg/reflect/type.go:/type. -struct Type -{ - uintptr size; - uint32 hash; - uint8 _unused; - uint8 align; - uint8 fieldAlign; - uint8 kind; - void* alg; - // gc stores type info required for garbage collector. - // If (kind&KindGCProg)==0, then gc[0] points at sparse GC bitmap - // (no indirection), 4 bits per word. - // If (kind&KindGCProg)!=0, then gc[1] points to a compiler-generated - // read-only GC program; and gc[0] points to BSS space for sparse GC bitmap. - // For huge types (>MaxGCMask), runtime unrolls the program directly into - // GC bitmap and gc[0] is not used. For moderately-sized types, runtime - // unrolls the program into gc[0] space on first use. The first byte of gc[0] - // (gc[0][0]) contains 'unroll' flag saying whether the program is already - // unrolled into gc[0] or not. - uintptr gc[2]; - String *string; - UncommonType *x; - Type *ptrto; - byte *zero; // ptr to the zero value for this type -}; - -struct Method -{ - String *name; - String *pkgPath; - Type *mtyp; - Type *typ; - void (*ifn)(void); - void (*tfn)(void); -}; - -struct UncommonType -{ - String *name; - String *pkgPath; - Slice mhdr; - Method m[]; -}; - -struct IMethod -{ - String *name; - String *pkgPath; - Type *type; -}; - -struct InterfaceType -{ - Type typ; - Slice mhdr; - IMethod m[]; -}; - -struct MapType -{ - Type typ; - Type *key; - Type *elem; - Type *bucket; // internal type representing a hash bucket - Type *hmap; // internal type representing a Hmap - uint8 keysize; // size of key slot - bool indirectkey; // store ptr to key instead of key itself - uint8 valuesize; // size of value slot - bool indirectvalue; // store ptr to value instead of value itself - uint16 bucketsize; // size of bucket -}; - -struct ChanType -{ - Type typ; - Type *elem; - uintptr dir; -}; - -struct SliceType -{ - Type typ; - Type *elem; -}; - -struct FuncType -{ - Type typ; - bool dotdotdot; - Slice in; - Slice out; -}; - -struct PtrType -{ - Type typ; - Type *elem; -}; diff --git a/src/runtime/typekind.h b/src/runtime/typekind.h index e0fe177bb..39cd45c2e 100644 --- a/src/runtime/typekind.h +++ b/src/runtime/typekind.h @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Must match runtime and reflect. +// Included by cmd/gc. + enum { KindBool = 1, KindInt, @@ -30,9 +33,8 @@ enum { KindStruct, KindUnsafePointer, - KindDirectIface = 1<<5, - KindGCProg = 1<<6, // Type.gc points to GC program - KindNoPointers = 1<<7, - KindMask = (1<<5)-1, + KindDirectIface = 1 << 5, + KindGCProg = 1 << 6, // Type.gc points to GC program + KindNoPointers = 1 << 7, + KindMask = (1 << 5) - 1, }; - diff --git a/src/runtime/typekind1.go b/src/runtime/typekind1.go new file mode 100644 index 000000000..73028d6f4 --- /dev/null +++ b/src/runtime/typekind1.go @@ -0,0 +1,39 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _KindBool = 1 + iota + _KindInt + _KindInt8 + _KindInt16 + _KindInt32 + _KindInt64 + _KindUint + _KindUint8 + _KindUint16 + _KindUint32 + _KindUint64 + _KindUintptr + _KindFloat32 + _KindFloat64 + _KindComplex64 + _KindComplex128 + _KindArray + _KindChan + _KindFunc + _KindInterface + _KindMap + _KindPtr + _KindSlice + _KindString + _KindStruct + _KindUnsafePointer + + _KindDirectIface = 1 << 5 + _KindGCProg = 1 << 6 // Type.gc points to GC program + _KindNoPointers = 1 << 7 + _KindMask = (1 << 5) - 1 +) diff --git a/src/runtime/vdso_linux_amd64.c b/src/runtime/vdso_linux_amd64.c deleted file mode 100644 index 681340c5b..000000000 --- a/src/runtime/vdso_linux_amd64.c +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "textflag.h" - -// Look up symbols in the Linux vDSO. - -// This code was originally based on the sample Linux vDSO parser at -// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/vDSO/parse_vdso.c - -// This implements the ELF dynamic linking spec at -// http://sco.com/developers/gabi/latest/ch5.dynamic.html - -// The version section is documented at -// http://refspecs.linuxfoundation.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/symversion.html - -#define AT_RANDOM 25 -#define AT_SYSINFO_EHDR 33 -#define AT_NULL 0 /* End of vector */ -#define PT_LOAD 1 /* Loadable program segment */ -#define PT_DYNAMIC 2 /* Dynamic linking information */ -#define DT_NULL 0 /* Marks end of dynamic section */ -#define DT_HASH 4 /* Dynamic symbol hash table */ -#define DT_STRTAB 5 /* Address of string table */ -#define DT_SYMTAB 6 /* Address of symbol table */ -#define DT_VERSYM 0x6ffffff0 -#define DT_VERDEF 0x6ffffffc - -#define VER_FLG_BASE 0x1 /* Version definition of file itself */ -#define SHN_UNDEF 0 /* Undefined section */ -#define SHT_DYNSYM 11 /* Dynamic linker symbol table */ -#define STT_FUNC 2 /* Symbol is a code object */ -#define STB_GLOBAL 1 /* Global symbol */ -#define STB_WEAK 2 /* Weak symbol */ - -/* How to extract and insert information held in the st_info field. */ -#define ELF64_ST_BIND(val) (((byte) (val)) >> 4) -#define ELF64_ST_TYPE(val) ((val) & 0xf) - -#define EI_NIDENT (16) - -typedef uint16 Elf64_Half; -typedef uint32 Elf64_Word; -typedef int32 Elf64_Sword; -typedef uint64 Elf64_Xword; -typedef int64 Elf64_Sxword; -typedef uint64 Elf64_Addr; -typedef uint64 Elf64_Off; -typedef uint16 Elf64_Section; -typedef Elf64_Half Elf64_Versym; - - -typedef struct Elf64_Sym -{ - Elf64_Word st_name; - byte st_info; - byte st_other; - Elf64_Section st_shndx; - Elf64_Addr st_value; - Elf64_Xword st_size; -} Elf64_Sym; - -typedef struct Elf64_Verdef -{ - Elf64_Half vd_version; /* Version revision */ - Elf64_Half vd_flags; /* Version information */ - Elf64_Half vd_ndx; /* Version Index */ - Elf64_Half vd_cnt; /* Number of associated aux entries */ - Elf64_Word vd_hash; /* Version name hash value */ - Elf64_Word vd_aux; /* Offset in bytes to verdaux array */ - Elf64_Word vd_next; /* Offset in bytes to next verdef entry */ -} Elf64_Verdef; - -typedef struct Elf64_Ehdr -{ - byte e_ident[EI_NIDENT]; /* Magic number and other info */ - Elf64_Half e_type; /* Object file type */ - Elf64_Half e_machine; /* Architecture */ - Elf64_Word e_version; /* Object file version */ - Elf64_Addr e_entry; /* Entry point virtual address */ - Elf64_Off e_phoff; /* Program header table file offset */ - Elf64_Off e_shoff; /* Section header table file offset */ - Elf64_Word e_flags; /* Processor-specific flags */ - Elf64_Half e_ehsize; /* ELF header size in bytes */ - Elf64_Half e_phentsize; /* Program header table entry size */ - Elf64_Half e_phnum; /* Program header table entry count */ - Elf64_Half e_shentsize; /* Section header table entry size */ - Elf64_Half e_shnum; /* Section header table entry count */ - Elf64_Half e_shstrndx; /* Section header string table index */ -} Elf64_Ehdr; - -typedef struct Elf64_Phdr -{ - Elf64_Word p_type; /* Segment type */ - Elf64_Word p_flags; /* Segment flags */ - Elf64_Off p_offset; /* Segment file offset */ - Elf64_Addr p_vaddr; /* Segment virtual address */ - Elf64_Addr p_paddr; /* Segment physical address */ - Elf64_Xword p_filesz; /* Segment size in file */ - Elf64_Xword p_memsz; /* Segment size in memory */ - Elf64_Xword p_align; /* Segment alignment */ -} Elf64_Phdr; - -typedef struct Elf64_Shdr -{ - Elf64_Word sh_name; /* Section name (string tbl index) */ - Elf64_Word sh_type; /* Section type */ - Elf64_Xword sh_flags; /* Section flags */ - Elf64_Addr sh_addr; /* Section virtual addr at execution */ - Elf64_Off sh_offset; /* Section file offset */ - Elf64_Xword sh_size; /* Section size in bytes */ - Elf64_Word sh_link; /* Link to another section */ - Elf64_Word sh_info; /* Additional section information */ - Elf64_Xword sh_addralign; /* Section alignment */ - Elf64_Xword sh_entsize; /* Entry size if section holds table */ -} Elf64_Shdr; - -typedef struct Elf64_Dyn -{ - Elf64_Sxword d_tag; /* Dynamic entry type */ - union - { - Elf64_Xword d_val; /* Integer value */ - Elf64_Addr d_ptr; /* Address value */ - } d_un; -} Elf64_Dyn; - -typedef struct Elf64_Verdaux -{ - Elf64_Word vda_name; /* Version or dependency names */ - Elf64_Word vda_next; /* Offset in bytes to next verdaux entry */ -} Elf64_Verdaux; - -typedef struct Elf64_auxv_t -{ - uint64 a_type; /* Entry type */ - union - { - uint64 a_val; /* Integer value */ - } a_un; -} Elf64_auxv_t; - - -typedef struct symbol_key { - byte* name; - int32 sym_hash; - void** var_ptr; -} symbol_key; - -typedef struct version_key { - byte* version; - int32 ver_hash; -} version_key; - -struct vdso_info { - bool valid; - - /* Load information */ - uintptr load_addr; - uintptr load_offset; /* load_addr - recorded vaddr */ - - /* Symbol table */ - Elf64_Sym *symtab; - const byte *symstrings; - Elf64_Word *bucket, *chain; - Elf64_Word nbucket, nchain; - - /* Version table */ - Elf64_Versym *versym; - Elf64_Verdef *verdef; -}; - -#pragma dataflag NOPTR -static version_key linux26 = { (byte*)"LINUX_2.6", 0x3ae75f6 }; - -// initialize with vsyscall fallbacks -#pragma dataflag NOPTR -void* runtime·__vdso_time_sym = (void*)0xffffffffff600400ULL; -#pragma dataflag NOPTR -void* runtime·__vdso_gettimeofday_sym = (void*)0xffffffffff600000ULL; -#pragma dataflag NOPTR -void* runtime·__vdso_clock_gettime_sym = (void*)0; - -#pragma dataflag NOPTR -static symbol_key sym_keys[] = { - { (byte*)"__vdso_time", 0xa33c485, &runtime·__vdso_time_sym }, - { (byte*)"__vdso_gettimeofday", 0x315ca59, &runtime·__vdso_gettimeofday_sym }, - { (byte*)"__vdso_clock_gettime", 0xd35ec75, &runtime·__vdso_clock_gettime_sym }, -}; - -static void -vdso_init_from_sysinfo_ehdr(struct vdso_info *vdso_info, Elf64_Ehdr* hdr) -{ - uint64 i; - bool found_vaddr = false; - Elf64_Phdr *pt; - Elf64_Dyn *dyn; - Elf64_Word *hash; - - vdso_info->valid = false; - vdso_info->load_addr = (uintptr) hdr; - - pt = (Elf64_Phdr*)(vdso_info->load_addr + hdr->e_phoff); - dyn = nil; - - // We need two things from the segment table: the load offset - // and the dynamic table. - for(i=0; i<hdr->e_phnum; i++) { - if(pt[i].p_type == PT_LOAD && found_vaddr == false) { - found_vaddr = true; - vdso_info->load_offset = (uintptr)hdr - + (uintptr)pt[i].p_offset - - (uintptr)pt[i].p_vaddr; - } else if(pt[i].p_type == PT_DYNAMIC) { - dyn = (Elf64_Dyn*)((uintptr)hdr + pt[i].p_offset); - } - } - - if(found_vaddr == false || dyn == nil) - return; // Failed - - // Fish out the useful bits of the dynamic table. - hash = nil; - vdso_info->symstrings = nil; - vdso_info->symtab = nil; - vdso_info->versym = nil; - vdso_info->verdef = nil; - for(i=0; dyn[i].d_tag!=DT_NULL; i++) { - switch(dyn[i].d_tag) { - case DT_STRTAB: - vdso_info->symstrings = (const byte *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_SYMTAB: - vdso_info->symtab = (Elf64_Sym *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_HASH: - hash = (Elf64_Word *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_VERSYM: - vdso_info->versym = (Elf64_Versym *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_VERDEF: - vdso_info->verdef = (Elf64_Verdef *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - } - } - if(vdso_info->symstrings == nil || vdso_info->symtab == nil || hash == nil) - return; // Failed - - if(vdso_info->verdef == nil) - vdso_info->versym = 0; - - // Parse the hash table header. - vdso_info->nbucket = hash[0]; - vdso_info->nchain = hash[1]; - vdso_info->bucket = &hash[2]; - vdso_info->chain = &hash[vdso_info->nbucket + 2]; - - // That's all we need. - vdso_info->valid = true; -} - -static int32 -vdso_find_version(struct vdso_info *vdso_info, version_key* ver) -{ - if(vdso_info->valid == false) { - return 0; - } - Elf64_Verdef *def = vdso_info->verdef; - while(true) { - if((def->vd_flags & VER_FLG_BASE) == 0) { - Elf64_Verdaux *aux = (Elf64_Verdaux*)((byte *)def + def->vd_aux); - if(def->vd_hash == ver->ver_hash && - runtime·strcmp(ver->version, vdso_info->symstrings + aux->vda_name) == 0) { - return def->vd_ndx & 0x7fff; - } - } - - if(def->vd_next == 0) { - break; - } - def = (Elf64_Verdef *)((byte *)def + def->vd_next); - } - return -1; // can not match any version -} - -static void -vdso_parse_symbols(struct vdso_info *vdso_info, int32 version) -{ - int32 i; - Elf64_Word chain; - Elf64_Sym *sym; - - if(vdso_info->valid == false) - return; - - for(i=0; i<nelem(sym_keys); i++) { - for(chain = vdso_info->bucket[sym_keys[i].sym_hash % vdso_info->nbucket]; - chain != 0; chain = vdso_info->chain[chain]) { - - sym = &vdso_info->symtab[chain]; - if(ELF64_ST_TYPE(sym->st_info) != STT_FUNC) - continue; - if(ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && - ELF64_ST_BIND(sym->st_info) != STB_WEAK) - continue; - if(sym->st_shndx == SHN_UNDEF) - continue; - if(runtime·strcmp(sym_keys[i].name, vdso_info->symstrings + sym->st_name) != 0) - continue; - - // Check symbol version. - if(vdso_info->versym != nil && version != 0 - && vdso_info->versym[chain] & 0x7fff != version) - continue; - - *sym_keys[i].var_ptr = (void *)(vdso_info->load_offset + sym->st_value); - break; - } - } -} - -static void -runtime·linux_setup_vdso(int32 argc, uint8** argv) -{ - struct vdso_info vdso_info; - - // skip argvc - byte **p = argv; - p = &p[argc+1]; - - // skip envp to get to ELF auxiliary vector. - for(; *p!=0; p++) {} - - // skip NULL separator - p++; - - // now, p points to auxv - Elf64_auxv_t *elf_auxv = (Elf64_auxv_t*) p; - - for(int32 i=0; elf_auxv[i].a_type!=AT_NULL; i++) { - if(elf_auxv[i].a_type == AT_SYSINFO_EHDR) { - if(elf_auxv[i].a_un.a_val == 0) { - // Something went wrong - continue; - } - vdso_init_from_sysinfo_ehdr(&vdso_info, (Elf64_Ehdr*)elf_auxv[i].a_un.a_val); - vdso_parse_symbols(&vdso_info, vdso_find_version(&vdso_info, &linux26)); - continue; - } - if(elf_auxv[i].a_type == AT_RANDOM) { - runtime·startup_random_data = (byte*)elf_auxv[i].a_un.a_val; - runtime·startup_random_data_len = 16; - continue; - } - } -} - -void (*runtime·sysargs)(int32, uint8**) = runtime·linux_setup_vdso; diff --git a/src/runtime/vdso_linux_amd64.go b/src/runtime/vdso_linux_amd64.go new file mode 100644 index 000000000..7eb698811 --- /dev/null +++ b/src/runtime/vdso_linux_amd64.go @@ -0,0 +1,328 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Look up symbols in the Linux vDSO. + +// This code was originally based on the sample Linux vDSO parser at +// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/vDSO/parse_vdso.c + +// This implements the ELF dynamic linking spec at +// http://sco.com/developers/gabi/latest/ch5.dynamic.html + +// The version section is documented at +// http://refspecs.linuxfoundation.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/symversion.html + +const ( + _AT_RANDOM = 25 + _AT_SYSINFO_EHDR = 33 + _AT_NULL = 0 /* End of vector */ + + _PT_LOAD = 1 /* Loadable program segment */ + _PT_DYNAMIC = 2 /* Dynamic linking information */ + + _DT_NULL = 0 /* Marks end of dynamic section */ + _DT_HASH = 4 /* Dynamic symbol hash table */ + _DT_STRTAB = 5 /* Address of string table */ + _DT_SYMTAB = 6 /* Address of symbol table */ + _DT_VERSYM = 0x6ffffff0 + _DT_VERDEF = 0x6ffffffc + + _VER_FLG_BASE = 0x1 /* Version definition of file itself */ + + _SHN_UNDEF = 0 /* Undefined section */ + + _SHT_DYNSYM = 11 /* Dynamic linker symbol table */ + + _STT_FUNC = 2 /* Symbol is a code object */ + + _STB_GLOBAL = 1 /* Global symbol */ + _STB_WEAK = 2 /* Weak symbol */ + + _EI_NIDENT = 16 +) + +/* How to extract and insert information held in the st_info field. */ +func _ELF64_ST_BIND(val byte) byte { return val >> 4 } +func _ELF64_ST_TYPE(val byte) byte { return val & 0xf } + +type elf64Sym struct { + st_name uint32 + st_info byte + st_other byte + st_shndx uint16 + st_value uint64 + st_size uint64 +} + +type elf64Verdef struct { + vd_version uint16 /* Version revision */ + vd_flags uint16 /* Version information */ + vd_ndx uint16 /* Version Index */ + vd_cnt uint16 /* Number of associated aux entries */ + vd_hash uint32 /* Version name hash value */ + vd_aux uint32 /* Offset in bytes to verdaux array */ + vd_next uint32 /* Offset in bytes to next verdef entry */ +} + +type elf64Ehdr struct { + e_ident [_EI_NIDENT]byte /* Magic number and other info */ + e_type uint16 /* Object file type */ + e_machine uint16 /* Architecture */ + e_version uint32 /* Object file version */ + e_entry uint64 /* Entry point virtual address */ + e_phoff uint64 /* Program header table file offset */ + e_shoff uint64 /* Section header table file offset */ + e_flags uint32 /* Processor-specific flags */ + e_ehsize uint16 /* ELF header size in bytes */ + e_phentsize uint16 /* Program header table entry size */ + e_phnum uint16 /* Program header table entry count */ + e_shentsize uint16 /* Section header table entry size */ + e_shnum uint16 /* Section header table entry count */ + e_shstrndx uint16 /* Section header string table index */ +} + +type elf64Phdr struct { + p_type uint32 /* Segment type */ + p_flags uint32 /* Segment flags */ + p_offset uint64 /* Segment file offset */ + p_vaddr uint64 /* Segment virtual address */ + p_paddr uint64 /* Segment physical address */ + p_filesz uint64 /* Segment size in file */ + p_memsz uint64 /* Segment size in memory */ + p_align uint64 /* Segment alignment */ +} + +type elf64Shdr struct { + sh_name uint32 /* Section name (string tbl index) */ + sh_type uint32 /* Section type */ + sh_flags uint64 /* Section flags */ + sh_addr uint64 /* Section virtual addr at execution */ + sh_offset uint64 /* Section file offset */ + sh_size uint64 /* Section size in bytes */ + sh_link uint32 /* Link to another section */ + sh_info uint32 /* Additional section information */ + sh_addralign uint64 /* Section alignment */ + sh_entsize uint64 /* Entry size if section holds table */ +} + +type elf64Dyn struct { + d_tag int64 /* Dynamic entry type */ + d_val uint64 /* Integer value */ +} + +type elf64Verdaux struct { + vda_name uint32 /* Version or dependency names */ + vda_next uint32 /* Offset in bytes to next verdaux entry */ +} + +type elf64Auxv struct { + a_type uint64 /* Entry type */ + a_val uint64 /* Integer value */ +} + +type symbol_key struct { + name string + sym_hash uint32 + ptr *uintptr +} + +type version_key struct { + version string + ver_hash uint32 +} + +type vdso_info struct { + valid bool + + /* Load information */ + load_addr uintptr + load_offset uintptr /* load_addr - recorded vaddr */ + + /* Symbol table */ + symtab *[1 << 32]elf64Sym + symstrings *[1 << 32]byte + chain []uint32 + bucket []uint32 + + /* Version table */ + versym *[1 << 32]uint16 + verdef *elf64Verdef +} + +var linux26 = version_key{"LINUX_2.6", 0x3ae75f6} + +var sym_keys = []symbol_key{ + {"__vdso_time", 0xa33c485, &__vdso_time_sym}, + {"__vdso_gettimeofday", 0x315ca59, &__vdso_gettimeofday_sym}, + {"__vdso_clock_gettime", 0xd35ec75, &__vdso_clock_gettime_sym}, +} + +// initialize with vsyscall fallbacks +var ( + __vdso_time_sym uintptr = 0xffffffffff600400 + __vdso_gettimeofday_sym uintptr = 0xffffffffff600000 + __vdso_clock_gettime_sym uintptr = 0 +) + +func vdso_init_from_sysinfo_ehdr(info *vdso_info, hdr *elf64Ehdr) { + info.valid = false + info.load_addr = uintptr(unsafe.Pointer(hdr)) + + pt := unsafe.Pointer(info.load_addr + uintptr(hdr.e_phoff)) + + // We need two things from the segment table: the load offset + // and the dynamic table. + var found_vaddr bool + var dyn *[1 << 20]elf64Dyn + for i := uint16(0); i < hdr.e_phnum; i++ { + pt := (*elf64Phdr)(add(pt, uintptr(i)*unsafe.Sizeof(elf64Phdr{}))) + switch pt.p_type { + case _PT_LOAD: + if !found_vaddr { + found_vaddr = true + info.load_offset = info.load_addr + uintptr(pt.p_offset-pt.p_vaddr) + } + + case _PT_DYNAMIC: + dyn = (*[1 << 20]elf64Dyn)(unsafe.Pointer(info.load_addr + uintptr(pt.p_offset))) + } + } + + if !found_vaddr || dyn == nil { + return // Failed + } + + // Fish out the useful bits of the dynamic table. + + var hash *[1 << 30]uint32 + hash = nil + info.symstrings = nil + info.symtab = nil + info.versym = nil + info.verdef = nil + for i := 0; dyn[i].d_tag != _DT_NULL; i++ { + dt := &dyn[i] + p := info.load_offset + uintptr(dt.d_val) + switch dt.d_tag { + case _DT_STRTAB: + info.symstrings = (*[1 << 32]byte)(unsafe.Pointer(p)) + case _DT_SYMTAB: + info.symtab = (*[1 << 32]elf64Sym)(unsafe.Pointer(p)) + case _DT_HASH: + hash = (*[1 << 30]uint32)(unsafe.Pointer(p)) + case _DT_VERSYM: + info.versym = (*[1 << 32]uint16)(unsafe.Pointer(p)) + case _DT_VERDEF: + info.verdef = (*elf64Verdef)(unsafe.Pointer(p)) + } + } + + if info.symstrings == nil || info.symtab == nil || hash == nil { + return // Failed + } + + if info.verdef == nil { + info.versym = nil + } + + // Parse the hash table header. + nbucket := hash[0] + nchain := hash[1] + info.bucket = hash[2 : 2+nbucket] + info.chain = hash[2+nbucket : 2+nbucket+nchain] + + // That's all we need. + info.valid = true +} + +func vdso_find_version(info *vdso_info, ver *version_key) int32 { + if !info.valid { + return 0 + } + + def := info.verdef + for { + if def.vd_flags&_VER_FLG_BASE == 0 { + aux := (*elf64Verdaux)(add(unsafe.Pointer(def), uintptr(def.vd_aux))) + if def.vd_hash == ver.ver_hash && ver.version == gostringnocopy(&info.symstrings[aux.vda_name]) { + return int32(def.vd_ndx & 0x7fff) + } + } + + if def.vd_next == 0 { + break + } + def = (*elf64Verdef)(add(unsafe.Pointer(def), uintptr(def.vd_next))) + } + + return -1 // can not match any version +} + +func vdso_parse_symbols(info *vdso_info, version int32) { + if !info.valid { + return + } + + for _, k := range sym_keys { + for chain := info.bucket[k.sym_hash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] { + sym := &info.symtab[chain] + typ := _ELF64_ST_TYPE(sym.st_info) + bind := _ELF64_ST_BIND(sym.st_info) + if typ != _STT_FUNC || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF { + continue + } + if k.name != gostringnocopy(&info.symstrings[sym.st_name]) { + continue + } + + // Check symbol version. + if info.versym != nil && version != 0 && int32(info.versym[chain]&0x7fff) != version { + continue + } + + *k.ptr = info.load_offset + uintptr(sym.st_value) + break + } + } +} + +func sysargs(argc int32, argv **byte) { + n := argc + 1 + + // skip envp to get to ELF auxiliary vector. + for argv_index(argv, n) != nil { + n++ + } + + // skip NULL separator + n++ + + // now argv+n is auxv + auxv := (*[1 << 32]elf64Auxv)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i].a_type != _AT_NULL; i++ { + av := &auxv[i] + switch av.a_type { + case _AT_SYSINFO_EHDR: + if av.a_val == 0 { + // Something went wrong + continue + } + var info vdso_info + // TODO(rsc): I don't understand why the compiler thinks info escapes + // when passed to the three functions below. + info1 := (*vdso_info)(noescape(unsafe.Pointer(&info))) + vdso_init_from_sysinfo_ehdr(info1, (*elf64Ehdr)(unsafe.Pointer(uintptr(av.a_val)))) + vdso_parse_symbols(info1, vdso_find_version(info1, &linux26)) + + case _AT_RANDOM: + startup_random_data = (*byte)(unsafe.Pointer(uintptr(av.a_val))) + startup_random_data_len = 16 + } + } +} diff --git a/src/runtime/vdso_none.go b/src/runtime/vdso_none.go new file mode 100644 index 000000000..ac6f8cb18 --- /dev/null +++ b/src/runtime/vdso_none.go @@ -0,0 +1,10 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux !amd64 + +package runtime + +func sysargs(argc int32, argv **byte) { +} diff --git a/src/runtime/vlop_arm.s b/src/runtime/vlop_arm.s index b4b905bb7..5354bf911 100644 --- a/src/runtime/vlop_arm.s +++ b/src/runtime/vlop_arm.s @@ -23,7 +23,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "textflag.h" arg=0 @@ -100,7 +101,7 @@ TEXT _sfloat(SB), NOSPLIT, $68-0 // 4 arg + 14*4 saved regs + cpsr + return valu // load the signal fault address into LR, and jump // to the real sigpanic. // This simulates what sighandler does for a memory fault. -TEXT _sfloatpanic(SB),NOSPLIT,$-4 +TEXT runtime·_sfloatpanic(SB),NOSPLIT,$-4 MOVW $0, R0 MOVW.W R0, -4(R13) MOVW g_sigpc(g), LR diff --git a/src/runtime/vlrt.c b/src/runtime/vlrt.c deleted file mode 100644 index cb0d14796..000000000 --- a/src/runtime/vlrt.c +++ /dev/null @@ -1,914 +0,0 @@ -// Inferno's libkern/vlrt-386.c -// http://code.google.com/p/inferno-os/source/browse/libkern/vlrt-386.c -// -// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. -// Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. -// Portions Copyright 2009 The Go Authors. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// +build arm 386 - -#include "textflag.h" - -/* - * C runtime for 64-bit divide, others. - * - * TODO(rsc): The simple functions are dregs--8c knows how - * to generate the code directly now. Find and remove. - */ - -void runtime·panicdivide(void); - -typedef unsigned long ulong; -typedef unsigned int uint; -typedef unsigned short ushort; -typedef unsigned char uchar; -typedef signed char schar; - -#define SIGN(n) (1UL<<(n-1)) - -typedef struct Vlong Vlong; -struct Vlong -{ - ulong lo; - ulong hi; -}; - -typedef union Vlong64 Vlong64; -union Vlong64 -{ - long long v; - Vlong v2; -}; - -void runtime·abort(void); - -#pragma textflag NOSPLIT -Vlong -_addv(Vlong a, Vlong b) -{ - Vlong r; - - r.lo = a.lo + b.lo; - r.hi = a.hi + b.hi; - if(r.lo < a.lo) - r.hi++; - return r; -} - -#pragma textflag NOSPLIT -Vlong -_subv(Vlong a, Vlong b) -{ - Vlong r; - - r.lo = a.lo - b.lo; - r.hi = a.hi - b.hi; - if(r.lo > a.lo) - r.hi--; - return r; -} - -Vlong -_d2v(double d) -{ - union { double d; Vlong vl; } x; - ulong xhi, xlo, ylo, yhi; - int sh; - Vlong y; - - x.d = d; - - xhi = (x.vl.hi & 0xfffff) | 0x100000; - xlo = x.vl.lo; - sh = 1075 - ((x.vl.hi >> 20) & 0x7ff); - - ylo = 0; - yhi = 0; - if(sh >= 0) { - /* v = (hi||lo) >> sh */ - if(sh < 32) { - if(sh == 0) { - ylo = xlo; - yhi = xhi; - } else { - ylo = (xlo >> sh) | (xhi << (32-sh)); - yhi = xhi >> sh; - } - } else { - if(sh == 32) { - ylo = xhi; - } else - if(sh < 64) { - ylo = xhi >> (sh-32); - } - } - } else { - /* v = (hi||lo) << -sh */ - sh = -sh; - if(sh <= 10) { /* NOTE: sh <= 11 on ARM??? */ - ylo = xlo << sh; - yhi = (xhi << sh) | (xlo >> (32-sh)); - } else { - /* overflow */ - yhi = d; /* causes something awful */ - } - } - if(x.vl.hi & SIGN(32)) { - if(ylo != 0) { - ylo = -ylo; - yhi = ~yhi; - } else - yhi = -yhi; - } - - y.hi = yhi; - y.lo = ylo; - return y; -} - -Vlong -_f2v(float f) -{ - return _d2v(f); -} - -double -_ul2d(ulong u) -{ - // compensate for bug in c - if(u & SIGN(32)) { - u ^= SIGN(32); - return 2147483648. + u; - } - return u; -} - -double -_v2d(Vlong x) -{ - if(x.hi & SIGN(32)) { - if(x.lo) { - x.lo = -x.lo; - x.hi = ~x.hi; - } else - x.hi = -x.hi; - return -(_ul2d(x.hi)*4294967296. + _ul2d(x.lo)); - } - return (long)x.hi*4294967296. + x.lo; -} - -float -_v2f(Vlong x) -{ - return _v2d(x); -} - -ulong runtime·_div64by32(Vlong, ulong, ulong*); -int runtime·_mul64by32(Vlong*, Vlong, ulong); - -static void -slowdodiv(Vlong num, Vlong den, Vlong *q, Vlong *r) -{ - ulong numlo, numhi, denhi, denlo, quohi, quolo, t; - int i; - - numhi = num.hi; - numlo = num.lo; - denhi = den.hi; - denlo = den.lo; - - /* - * get a divide by zero - */ - if(denlo==0 && denhi==0) { - runtime·panicdivide(); - } - - /* - * set up the divisor and find the number of iterations needed - */ - if(numhi >= SIGN(32)) { - quohi = SIGN(32); - quolo = 0; - } else { - quohi = numhi; - quolo = numlo; - } - i = 0; - while(denhi < quohi || (denhi == quohi && denlo < quolo)) { - denhi = (denhi<<1) | (denlo>>31); - denlo <<= 1; - i++; - } - - quohi = 0; - quolo = 0; - for(; i >= 0; i--) { - quohi = (quohi<<1) | (quolo>>31); - quolo <<= 1; - if(numhi > denhi || (numhi == denhi && numlo >= denlo)) { - t = numlo; - numlo -= denlo; - if(numlo > t) - numhi--; - numhi -= denhi; - quolo |= 1; - } - denlo = (denlo>>1) | (denhi<<31); - denhi >>= 1; - } - - if(q) { - q->lo = quolo; - q->hi = quohi; - } - if(r) { - r->lo = numlo; - r->hi = numhi; - } -} - -#ifdef GOARCH_arm -static void -dodiv(Vlong num, Vlong den, Vlong *qp, Vlong *rp) -{ - slowdodiv(num, den, qp, rp); -} -#endif - -#ifdef GOARCH_386 -static void -dodiv(Vlong num, Vlong den, Vlong *qp, Vlong *rp) -{ - ulong n; - Vlong x, q, r; - - if(den.hi > num.hi || (den.hi == num.hi && den.lo > num.lo)){ - if(qp) { - qp->hi = 0; - qp->lo = 0; - } - if(rp) { - rp->hi = num.hi; - rp->lo = num.lo; - } - return; - } - - if(den.hi != 0){ - q.hi = 0; - n = num.hi/den.hi; - if(runtime·_mul64by32(&x, den, n) || x.hi > num.hi || (x.hi == num.hi && x.lo > num.lo)) - slowdodiv(num, den, &q, &r); - else { - q.lo = n; - *(long long*)&r = *(long long*)&num - *(long long*)&x; - } - } else { - if(num.hi >= den.lo){ - if(den.lo == 0) - runtime·panicdivide(); - q.hi = n = num.hi/den.lo; - num.hi -= den.lo*n; - } else { - q.hi = 0; - } - q.lo = runtime·_div64by32(num, den.lo, &r.lo); - r.hi = 0; - } - if(qp) { - qp->lo = q.lo; - qp->hi = q.hi; - } - if(rp) { - rp->lo = r.lo; - rp->hi = r.hi; - } -} -#endif - -Vlong -_divvu(Vlong n, Vlong d) -{ - Vlong q; - - if(n.hi == 0 && d.hi == 0) { - if(d.lo == 0) - runtime·panicdivide(); - q.hi = 0; - q.lo = n.lo / d.lo; - return q; - } - dodiv(n, d, &q, 0); - return q; -} - -Vlong -_modvu(Vlong n, Vlong d) -{ - Vlong r; - - if(n.hi == 0 && d.hi == 0) { - if(d.lo == 0) - runtime·panicdivide(); - r.hi = 0; - r.lo = n.lo % d.lo; - return r; - } - dodiv(n, d, 0, &r); - return r; -} - -static void -vneg(Vlong *v) -{ - - if(v->lo == 0) { - v->hi = -v->hi; - return; - } - v->lo = -v->lo; - v->hi = ~v->hi; -} - -Vlong -_divv(Vlong n, Vlong d) -{ - long nneg, dneg; - Vlong q; - - if(n.hi == (((long)n.lo)>>31) && d.hi == (((long)d.lo)>>31)) { - if((long)n.lo == -0x80000000 && (long)d.lo == -1) { - // special case: 32-bit -0x80000000 / -1 causes divide error, - // but it's okay in this 64-bit context. - q.lo = 0x80000000; - q.hi = 0; - return q; - } - if(d.lo == 0) - runtime·panicdivide(); - q.lo = (long)n.lo / (long)d.lo; - q.hi = ((long)q.lo) >> 31; - return q; - } - nneg = n.hi >> 31; - if(nneg) - vneg(&n); - dneg = d.hi >> 31; - if(dneg) - vneg(&d); - dodiv(n, d, &q, 0); - if(nneg != dneg) - vneg(&q); - return q; -} - -Vlong -_modv(Vlong n, Vlong d) -{ - long nneg, dneg; - Vlong r; - - if(n.hi == (((long)n.lo)>>31) && d.hi == (((long)d.lo)>>31)) { - if((long)n.lo == -0x80000000 && (long)d.lo == -1) { - // special case: 32-bit -0x80000000 % -1 causes divide error, - // but it's okay in this 64-bit context. - r.lo = 0; - r.hi = 0; - return r; - } - if(d.lo == 0) - runtime·panicdivide(); - r.lo = (long)n.lo % (long)d.lo; - r.hi = ((long)r.lo) >> 31; - return r; - } - nneg = n.hi >> 31; - if(nneg) - vneg(&n); - dneg = d.hi >> 31; - if(dneg) - vneg(&d); - dodiv(n, d, 0, &r); - if(nneg) - vneg(&r); - return r; -} - -#pragma textflag NOSPLIT -Vlong -_rshav(Vlong a, int b) -{ - long t; - Vlong r; - - t = a.hi; - if(b >= 32) { - r.hi = t>>31; - if(b >= 64) { - /* this is illegal re C standard */ - r.lo = t>>31; - return r; - } - r.lo = t >> (b-32); - return r; - } - if(b <= 0) { - r.hi = t; - r.lo = a.lo; - return r; - } - r.hi = t >> b; - r.lo = (t << (32-b)) | (a.lo >> b); - return r; -} - -#pragma textflag NOSPLIT -Vlong -_rshlv(Vlong a, int b) -{ - ulong t; - Vlong r; - - t = a.hi; - if(b >= 32) { - r.hi = 0; - if(b >= 64) { - /* this is illegal re C standard */ - r.lo = 0; - return r; - } - r.lo = t >> (b-32); - return r; - } - if(b <= 0) { - r.hi = t; - r.lo = a.lo; - return r; - } - r.hi = t >> b; - r.lo = (t << (32-b)) | (a.lo >> b); - return r; -} - -#pragma textflag NOSPLIT -Vlong -_lshv(Vlong a, int b) -{ - ulong t; - - t = a.lo; - if(b >= 32) { - if(b >= 64) { - /* this is illegal re C standard */ - return (Vlong){0, 0}; - } - return (Vlong){0, t<<(b-32)}; - } - if(b <= 0) { - return (Vlong){t, a.hi}; - } - return (Vlong){t<<b, (t >> (32-b)) | (a.hi << b)}; -} - -#pragma textflag NOSPLIT -Vlong -_andv(Vlong a, Vlong b) -{ - Vlong r; - - r.hi = a.hi & b.hi; - r.lo = a.lo & b.lo; - return r; -} - -#pragma textflag NOSPLIT -Vlong -_orv(Vlong a, Vlong b) -{ - Vlong r; - - r.hi = a.hi | b.hi; - r.lo = a.lo | b.lo; - return r; -} - -#pragma textflag NOSPLIT -Vlong -_xorv(Vlong a, Vlong b) -{ - Vlong r; - - r.hi = a.hi ^ b.hi; - r.lo = a.lo ^ b.lo; - return r; -} - -Vlong -_vpp(Vlong *r) -{ - Vlong l; - - l = *r; - r->lo++; - if(r->lo == 0) - r->hi++; - return l; -} - -#pragma textflag NOSPLIT -Vlong -_vmm(Vlong *r) -{ - Vlong l; - - l = *r; - if(r->lo == 0) - r->hi--; - r->lo--; - return l; -} - -#pragma textflag NOSPLIT -Vlong -_ppv(Vlong *r) -{ - - r->lo++; - if(r->lo == 0) - r->hi++; - return *r; -} - -#pragma textflag NOSPLIT -Vlong -_mmv(Vlong *r) -{ - - if(r->lo == 0) - r->hi--; - r->lo--; - return *r; -} - -#pragma textflag NOSPLIT -Vlong -_vasop(void *lv, Vlong fn(Vlong, Vlong), int type, Vlong rv) -{ - Vlong t, u; - - u.lo = 0; - u.hi = 0; - switch(type) { - default: - runtime·abort(); - break; - - case 1: /* schar */ - t.lo = *(schar*)lv; - t.hi = t.lo >> 31; - u = fn(t, rv); - *(schar*)lv = u.lo; - break; - - case 2: /* uchar */ - t.lo = *(uchar*)lv; - t.hi = 0; - u = fn(t, rv); - *(uchar*)lv = u.lo; - break; - - case 3: /* short */ - t.lo = *(short*)lv; - t.hi = t.lo >> 31; - u = fn(t, rv); - *(short*)lv = u.lo; - break; - - case 4: /* ushort */ - t.lo = *(ushort*)lv; - t.hi = 0; - u = fn(t, rv); - *(ushort*)lv = u.lo; - break; - - case 9: /* int */ - t.lo = *(int*)lv; - t.hi = t.lo >> 31; - u = fn(t, rv); - *(int*)lv = u.lo; - break; - - case 10: /* uint */ - t.lo = *(uint*)lv; - t.hi = 0; - u = fn(t, rv); - *(uint*)lv = u.lo; - break; - - case 5: /* long */ - t.lo = *(long*)lv; - t.hi = t.lo >> 31; - u = fn(t, rv); - *(long*)lv = u.lo; - break; - - case 6: /* ulong */ - t.lo = *(ulong*)lv; - t.hi = 0; - u = fn(t, rv); - *(ulong*)lv = u.lo; - break; - - case 7: /* vlong */ - case 8: /* uvlong */ - if((void*)fn == _lshv || (void*)fn == _rshav || (void*)fn == _rshlv) - u = ((Vlong(*)(Vlong,int))fn)(*(Vlong*)lv, *(int*)&rv); - else - u = fn(*(Vlong*)lv, rv); - *(Vlong*)lv = u; - break; - } - return u; -} - -#pragma textflag NOSPLIT -Vlong -_p2v(void *p) -{ - long t; - Vlong ret; - - t = (ulong)p; - ret.lo = t; - ret.hi = 0; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_sl2v(long sl) -{ - long t; - Vlong ret; - - t = sl; - ret.lo = t; - ret.hi = t >> 31; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_ul2v(ulong ul) -{ - long t; - Vlong ret; - - t = ul; - ret.lo = t; - ret.hi = 0; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_si2v(int si) -{ - return (Vlong){si, si>>31}; -} - -#pragma textflag NOSPLIT -Vlong -_ui2v(uint ui) -{ - long t; - Vlong ret; - - t = ui; - ret.lo = t; - ret.hi = 0; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_sh2v(long sh) -{ - long t; - Vlong ret; - - t = (sh << 16) >> 16; - ret.lo = t; - ret.hi = t >> 31; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_uh2v(ulong ul) -{ - long t; - Vlong ret; - - t = ul & 0xffff; - ret.lo = t; - ret.hi = 0; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_sc2v(long uc) -{ - long t; - Vlong ret; - - t = (uc << 24) >> 24; - ret.lo = t; - ret.hi = t >> 31; - return ret; -} - -#pragma textflag NOSPLIT -Vlong -_uc2v(ulong ul) -{ - long t; - Vlong ret; - - t = ul & 0xff; - ret.lo = t; - ret.hi = 0; - return ret; -} - -#pragma textflag NOSPLIT -long -_v2sc(Vlong rv) -{ - long t; - - t = rv.lo & 0xff; - return (t << 24) >> 24; -} - -#pragma textflag NOSPLIT -long -_v2uc(Vlong rv) -{ - - return rv.lo & 0xff; -} - -#pragma textflag NOSPLIT -long -_v2sh(Vlong rv) -{ - long t; - - t = rv.lo & 0xffff; - return (t << 16) >> 16; -} - -#pragma textflag NOSPLIT -long -_v2uh(Vlong rv) -{ - - return rv.lo & 0xffff; -} - -#pragma textflag NOSPLIT -long -_v2sl(Vlong rv) -{ - - return rv.lo; -} - -#pragma textflag NOSPLIT -long -_v2ul(Vlong rv) -{ - - return rv.lo; -} - -#pragma textflag NOSPLIT -long -_v2si(Vlong rv) -{ - return rv.lo; -} - -#pragma textflag NOSPLIT -long -_v2ui(Vlong rv) -{ - - return rv.lo; -} - -#pragma textflag NOSPLIT -int -_testv(Vlong rv) -{ - return rv.lo || rv.hi; -} - -#pragma textflag NOSPLIT -int -_eqv(Vlong lv, Vlong rv) -{ - return lv.lo == rv.lo && lv.hi == rv.hi; -} - -#pragma textflag NOSPLIT -int -_nev(Vlong lv, Vlong rv) -{ - return lv.lo != rv.lo || lv.hi != rv.hi; -} - -#pragma textflag NOSPLIT -int -_ltv(Vlong lv, Vlong rv) -{ - return (long)lv.hi < (long)rv.hi || - (lv.hi == rv.hi && lv.lo < rv.lo); -} - -#pragma textflag NOSPLIT -int -_lev(Vlong lv, Vlong rv) -{ - return (long)lv.hi < (long)rv.hi || - (lv.hi == rv.hi && lv.lo <= rv.lo); -} - -#pragma textflag NOSPLIT -int -_gtv(Vlong lv, Vlong rv) -{ - return (long)lv.hi > (long)rv.hi || - (lv.hi == rv.hi && lv.lo > rv.lo); -} - -#pragma textflag NOSPLIT -int -_gev(Vlong lv, Vlong rv) -{ - return (long)lv.hi > (long)rv.hi || - (lv.hi == rv.hi && lv.lo >= rv.lo); -} - -#pragma textflag NOSPLIT -int -_lov(Vlong lv, Vlong rv) -{ - return lv.hi < rv.hi || - (lv.hi == rv.hi && lv.lo < rv.lo); -} - -#pragma textflag NOSPLIT -int -_lsv(Vlong lv, Vlong rv) -{ - return lv.hi < rv.hi || - (lv.hi == rv.hi && lv.lo <= rv.lo); -} - -#pragma textflag NOSPLIT -int -_hiv(Vlong lv, Vlong rv) -{ - return lv.hi > rv.hi || - (lv.hi == rv.hi && lv.lo > rv.lo); -} - -#pragma textflag NOSPLIT -int -_hsv(Vlong lv, Vlong rv) -{ - return lv.hi > rv.hi || - (lv.hi == rv.hi && lv.lo >= rv.lo); -} diff --git a/src/runtime/zgoarch_386.go b/src/runtime/zgoarch_386.go new file mode 100644 index 000000000..057a746bb --- /dev/null +++ b/src/runtime/zgoarch_386.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `386` + +const goarch_386 = 1 +const goarch_amd64 = 0 +const goarch_amd64p32 = 0 +const goarch_arm = 0 +const goarch_power64 = 0 +const goarch_power64le = 0 diff --git a/src/runtime/zgoarch_amd64.go b/src/runtime/zgoarch_amd64.go new file mode 100644 index 000000000..a71240726 --- /dev/null +++ b/src/runtime/zgoarch_amd64.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `amd64` + +const goarch_386 = 0 +const goarch_amd64 = 1 +const goarch_amd64p32 = 0 +const goarch_arm = 0 +const goarch_power64 = 0 +const goarch_power64le = 0 diff --git a/src/runtime/zgoarch_amd64p32.go b/src/runtime/zgoarch_amd64p32.go new file mode 100644 index 000000000..2b6a142bb --- /dev/null +++ b/src/runtime/zgoarch_amd64p32.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `amd64p32` + +const goarch_386 = 0 +const goarch_amd64 = 0 +const goarch_amd64p32 = 1 +const goarch_arm = 0 +const goarch_power64 = 0 +const goarch_power64le = 0 diff --git a/src/runtime/zgoarch_arm.go b/src/runtime/zgoarch_arm.go new file mode 100644 index 000000000..403021005 --- /dev/null +++ b/src/runtime/zgoarch_arm.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `arm` + +const goarch_386 = 0 +const goarch_amd64 = 0 +const goarch_amd64p32 = 0 +const goarch_arm = 1 +const goarch_power64 = 0 +const goarch_power64le = 0 diff --git a/src/runtime/zgoarch_power64.go b/src/runtime/zgoarch_power64.go new file mode 100644 index 000000000..cc361f050 --- /dev/null +++ b/src/runtime/zgoarch_power64.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `power64` + +const goarch_386 = 0 +const goarch_amd64 = 0 +const goarch_amd64p32 = 0 +const goarch_arm = 0 +const goarch_power64 = 1 +const goarch_power64le = 0 diff --git a/src/runtime/zgoarch_power64le.go b/src/runtime/zgoarch_power64le.go new file mode 100644 index 000000000..41294e61b --- /dev/null +++ b/src/runtime/zgoarch_power64le.go @@ -0,0 +1,12 @@ +// generated by gengoos.go using 'go generate' + +package runtime + +const theGoarch = `power64le` + +const goarch_386 = 0 +const goarch_amd64 = 0 +const goarch_amd64p32 = 0 +const goarch_arm = 0 +const goarch_power64 = 0 +const goarch_power64le = 1 diff --git a/src/runtime/zgoos_android.go b/src/runtime/zgoos_android.go new file mode 100644 index 000000000..abfba808b --- /dev/null +++ b/src/runtime/zgoos_android.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build android + +package runtime + +const theGoos = `android` + +const goos_android = 1 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_darwin.go b/src/runtime/zgoos_darwin.go new file mode 100644 index 000000000..eb39b53dd --- /dev/null +++ b/src/runtime/zgoos_darwin.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build darwin + +package runtime + +const theGoos = `darwin` + +const goos_android = 0 +const goos_darwin = 1 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_dragonfly.go b/src/runtime/zgoos_dragonfly.go new file mode 100644 index 000000000..f6e839d3d --- /dev/null +++ b/src/runtime/zgoos_dragonfly.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build dragonfly + +package runtime + +const theGoos = `dragonfly` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 1 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_freebsd.go b/src/runtime/zgoos_freebsd.go new file mode 100644 index 000000000..3c47aef2a --- /dev/null +++ b/src/runtime/zgoos_freebsd.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build freebsd + +package runtime + +const theGoos = `freebsd` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 1 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_linux.go b/src/runtime/zgoos_linux.go new file mode 100644 index 000000000..5d899e3db --- /dev/null +++ b/src/runtime/zgoos_linux.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build linux + +package runtime + +const theGoos = `linux` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 1 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_nacl.go b/src/runtime/zgoos_nacl.go new file mode 100644 index 000000000..b5c4281fb --- /dev/null +++ b/src/runtime/zgoos_nacl.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build nacl + +package runtime + +const theGoos = `nacl` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 1 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_netbsd.go b/src/runtime/zgoos_netbsd.go new file mode 100644 index 000000000..b2e45222a --- /dev/null +++ b/src/runtime/zgoos_netbsd.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build netbsd + +package runtime + +const theGoos = `netbsd` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 1 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_openbsd.go b/src/runtime/zgoos_openbsd.go new file mode 100644 index 000000000..331c96dd6 --- /dev/null +++ b/src/runtime/zgoos_openbsd.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build openbsd + +package runtime + +const theGoos = `openbsd` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 1 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_plan9.go b/src/runtime/zgoos_plan9.go new file mode 100644 index 000000000..f29eb4523 --- /dev/null +++ b/src/runtime/zgoos_plan9.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build plan9 + +package runtime + +const theGoos = `plan9` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 1 +const goos_solaris = 0 +const goos_windows = 0 diff --git a/src/runtime/zgoos_solaris.go b/src/runtime/zgoos_solaris.go new file mode 100644 index 000000000..ac613db33 --- /dev/null +++ b/src/runtime/zgoos_solaris.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build solaris + +package runtime + +const theGoos = `solaris` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 1 +const goos_windows = 0 diff --git a/src/runtime/zgoos_windows.go b/src/runtime/zgoos_windows.go new file mode 100644 index 000000000..43710d862 --- /dev/null +++ b/src/runtime/zgoos_windows.go @@ -0,0 +1,19 @@ +// generated by gengoos.go using 'go generate' + +// +build windows + +package runtime + +const theGoos = `windows` + +const goos_android = 0 +const goos_darwin = 0 +const goos_dragonfly = 0 +const goos_freebsd = 0 +const goos_linux = 0 +const goos_nacl = 0 +const goos_netbsd = 0 +const goos_openbsd = 0 +const goos_plan9 = 0 +const goos_solaris = 0 +const goos_windows = 1 |