diff options
| author | kcc <kcc@138bc75d-0d04-0410-961f-82ee72b054a4> | 2013-11-04 21:33:31 +0000 |
|---|---|---|
| committer | kcc <kcc@138bc75d-0d04-0410-961f-82ee72b054a4> | 2013-11-04 21:33:31 +0000 |
| commit | 1e80ce4111e28463d870335befe7d99066b5971e (patch) | |
| tree | 7cfc103c9b6b4ce7ca19d39f91509a1b68819a63 /libsanitizer/tsan | |
| parent | 482026b63e8a488d6b7f0eab53fcbfe12c3309ae (diff) | |
| download | gcc-1e80ce4111e28463d870335befe7d99066b5971e.tar.gz | |
libsanitizer merge from upstream r191666
This may break gcc-asan on Mac, will follow up separately.
git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@204368 138bc75d-0d04-0410-961f-82ee72b054a4
Diffstat (limited to 'libsanitizer/tsan')
39 files changed, 2080 insertions, 1163 deletions
diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h index b2937a428f3..68936e02e4b 100644 --- a/libsanitizer/tsan/tsan_defs.h +++ b/libsanitizer/tsan/tsan_defs.h @@ -154,7 +154,6 @@ struct MD5Hash { MD5Hash md5_hash(const void *data, uptr size); struct ThreadState; -struct ThreadContext; struct Context; struct ReportStack; class ReportDesc; diff --git a/libsanitizer/tsan/tsan_fd.cc b/libsanitizer/tsan/tsan_fd.cc index a75d9bde08a..de852b185a6 100644 --- a/libsanitizer/tsan/tsan_fd.cc +++ b/libsanitizer/tsan/tsan_fd.cc @@ -72,13 +72,14 @@ static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) { uptr l1 = atomic_load(pl1, memory_order_consume); if (l1 == 0) { uptr size = kTableSizeL2 * sizeof(FdDesc); - void *p = internal_alloc(MBlockFD, size); + // We need this to reside in user memory to properly catch races on it. + void *p = user_alloc(thr, pc, size); internal_memset(p, 0, size); MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size); if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel)) l1 = (uptr)p; else - internal_free(p); + user_free(thr, pc, p); } return &((FdDesc*)l1)[fd % kTableSizeL2]; // NOLINT } @@ -157,9 +158,9 @@ void FdRelease(ThreadState *thr, uptr pc, int fd) { FdDesc *d = fddesc(thr, pc, fd); FdSync *s = d->sync; DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s); + MemoryRead(thr, pc, (uptr)d, kSizeLog8); if (s) Release(thr, pc, (uptr)s); - MemoryRead(thr, pc, (uptr)d, kSizeLog8); } void FdAccess(ThreadState *thr, uptr pc, int fd) { diff --git a/libsanitizer/tsan/tsan_flags.cc b/libsanitizer/tsan/tsan_flags.cc index ae748a13e15..dbde4212c59 100644 --- a/libsanitizer/tsan/tsan_flags.cc +++ b/libsanitizer/tsan/tsan_flags.cc @@ -47,17 +47,22 @@ void InitializeFlags(Flags *f, const char *env) { f->force_seq_cst_atomics = false; f->strip_path_prefix = ""; f->suppressions = ""; + f->print_suppressions = false; + f->print_benign = false; f->exitcode = 66; + f->halt_on_error = false; f->log_path = "stderr"; f->atexit_sleep_ms = 1000; f->verbosity = 0; f->profile_memory = ""; f->flush_memory_ms = 0; + f->flush_symbolizer_ms = 5000; f->stop_on_start = false; f->running_on_valgrind = false; f->external_symbolizer_path = ""; f->history_size = kGoMode ? 1 : 2; // There are a lot of goroutines in Go. f->io_sync = 1; + f->allocator_may_return_null = false; // Let a frontend override. OverrideFlags(f); @@ -75,16 +80,21 @@ void InitializeFlags(Flags *f, const char *env) { ParseFlag(env, &f->force_seq_cst_atomics, "force_seq_cst_atomics"); ParseFlag(env, &f->strip_path_prefix, "strip_path_prefix"); ParseFlag(env, &f->suppressions, "suppressions"); + ParseFlag(env, &f->print_suppressions, "print_suppressions"); + ParseFlag(env, &f->print_benign, "print_benign"); ParseFlag(env, &f->exitcode, "exitcode"); + ParseFlag(env, &f->halt_on_error, "halt_on_error"); ParseFlag(env, &f->log_path, "log_path"); ParseFlag(env, &f->atexit_sleep_ms, "atexit_sleep_ms"); ParseFlag(env, &f->verbosity, "verbosity"); ParseFlag(env, &f->profile_memory, "profile_memory"); ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms"); + ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms"); ParseFlag(env, &f->stop_on_start, "stop_on_start"); ParseFlag(env, &f->external_symbolizer_path, "external_symbolizer_path"); ParseFlag(env, &f->history_size, "history_size"); ParseFlag(env, &f->io_sync, "io_sync"); + ParseFlag(env, &f->allocator_may_return_null, "allocator_may_return_null"); if (!f->report_bugs) { f->report_thread_leaks = false; @@ -103,6 +113,8 @@ void InitializeFlags(Flags *f, const char *env) { " (must be [0..2])\n"); Die(); } + + common_flags()->allocator_may_return_null = f->allocator_may_return_null; } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_flags.h b/libsanitizer/tsan/tsan_flags.h index 480b41538f9..a7571c92b5e 100644 --- a/libsanitizer/tsan/tsan_flags.h +++ b/libsanitizer/tsan/tsan_flags.h @@ -50,8 +50,14 @@ struct Flags { const char *strip_path_prefix; // Suppressions filename. const char *suppressions; + // Print matched suppressions at exit. + bool print_suppressions; + // Print matched "benign" races at exit. + bool print_benign; // Override exit status if something was reported. int exitcode; + // Exit after first reported error. + bool halt_on_error; // Write logs to "log_path.pid". // The special values are "stdout" and "stderr". // The default is "stderr". @@ -65,6 +71,8 @@ struct Flags { const char *profile_memory; // Flush shadow memory every X ms. int flush_memory_ms; + // Flush symbolizer caches every X ms. + int flush_symbolizer_ms; // Stops on start until __tsan_resume() is called (for debugging). bool stop_on_start; // Controls whether RunningOnValgrind() returns true or false. @@ -82,6 +90,8 @@ struct Flags { // 1 - reasonable level of synchronization (write->read) // 2 - global synchronization of all IO operations int io_sync; + // If false, the allocator will crash instead of returning 0 on out-of-memory. + bool allocator_may_return_null; }; Flags *flags(); diff --git a/libsanitizer/tsan/tsan_interceptors.cc b/libsanitizer/tsan/tsan_interceptors.cc index 8a54511b6e0..eaaf9e3ebf6 100644 --- a/libsanitizer/tsan/tsan_interceptors.cc +++ b/libsanitizer/tsan/tsan_interceptors.cc @@ -8,11 +8,13 @@ // This file is a part of ThreadSanitizer (TSan), a race detector. // // FIXME: move as many interceptors as possible into -// sanitizer_common/sanitizer_common_interceptors.h +// sanitizer_common/sanitizer_common_interceptors.inc //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "interception/interception.h" @@ -24,18 +26,16 @@ using namespace __tsan; // NOLINT -const int kSigCount = 128; +const int kSigCount = 64; struct my_siginfo_t { - int opaque[128]; -}; - -struct sigset_t { - u64 val[1024 / 8 / sizeof(u64)]; + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; }; struct ucontext_t { - uptr opaque[117]; + // The size is determined by looking at sizeof of real ucontext_t on linux. + u64 opaque[936 / sizeof(u64) + 1]; }; extern "C" int pthread_attr_init(void *attr); @@ -47,8 +47,10 @@ extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); extern "C" int pthread_mutexattr_gettype(void *a, int *type); extern "C" int pthread_yield(); -extern "C" int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); -extern "C" int sigfillset(sigset_t *set); +extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset); +// REAL(sigfillset) defined in common interceptors. +DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); @@ -57,9 +59,9 @@ extern "C" void *__libc_malloc(uptr size); extern "C" void *__libc_calloc(uptr size, uptr n); extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); +extern "C" int mallopt(int param, int value); const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; -const int kPthreadAttrSize = 56; const int EINVAL = 22; const int EBUSY = 16; const int EPOLL_CTL_ADD = 1; @@ -69,6 +71,7 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGBUS = 7; +const int SIGSYS = 31; void *const MAP_FAILED = (void*)-1; const int PTHREAD_BARRIER_SERIAL_THREAD = -1; const int MAP_FIXED = 0x10; @@ -84,17 +87,12 @@ typedef void (*sighandler_t)(int sig); #define errno (*__errno_location()) -union pthread_attr_t { - char size[kPthreadAttrSize]; - void *align; -}; - struct sigaction_t { union { sighandler_t sa_handler; void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); }; - sigset_t sa_mask; + __sanitizer_sigset_t sa_mask; int sa_flags; void (*sa_restorer)(); }; @@ -125,6 +123,21 @@ struct SignalContext { int pending_signal_count; SignalDesc pending_signals[kSigCount]; }; + +// Used to ignore interceptors coming directly from libjvm.so. +atomic_uintptr_t libjvm_begin; +atomic_uintptr_t libjvm_end; + +static bool libjvm_check(uptr pc) { + uptr begin = atomic_load(&libjvm_begin, memory_order_relaxed); + if (begin != 0 && pc >= begin) { + uptr end = atomic_load(&libjvm_end, memory_order_relaxed); + if (end != 0 && pc < end) + return true; + } + return false; +} + } // namespace __tsan static SignalContext *SigCtx(ThreadState *thr) { @@ -178,8 +191,7 @@ ScopedInterceptor::~ScopedInterceptor() { StatInc(thr, StatInt_##func); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ - const uptr pc = __sanitizer::StackTrace::GetPreviousInstructionPc( \ - __sanitizer::StackTrace::GetCurrentPc()); \ + const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ (void)pc; \ /**/ @@ -189,7 +201,7 @@ ScopedInterceptor::~ScopedInterceptor() { Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ Die(); \ } \ - if (thr->in_rtl > 1) \ + if (thr->in_rtl > 1 || libjvm_check(pc)) \ return REAL(func)(__VA_ARGS__); \ /**/ @@ -293,15 +305,6 @@ class AtExitContext { static AtExitContext *atexit_ctx; -static void finalize(void *arg) { - ThreadState * thr = cur_thread(); - uptr pc = 0; - atexit_ctx->exit(thr, pc); - int status = Finalize(cur_thread()); - if (status) - _exit(status); -} - TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (cur_thread()->in_symbolizer) return 0; @@ -320,25 +323,123 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); - if (dso) - return REAL(__cxa_atexit)(f, arg, dso); + if (dso) { + // Memory allocation in __cxa_atexit will race with free during exit, + // because we do not see synchronization around atexit callback list. + ThreadIgnoreBegin(thr); + int res = REAL(__cxa_atexit)(f, arg, dso); + ThreadIgnoreEnd(thr); + return res; + } return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg); } -TSAN_INTERCEPTOR(void, longjmp, void *env, int val) { - SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); - Printf("ThreadSanitizer: longjmp() is not supported\n"); - Die(); +// Cleanup old bufs. +static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { + for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { + JmpBuf *buf = &thr->jmp_bufs[i]; + if (buf->sp <= sp) { + uptr sz = thr->jmp_bufs.Size(); + thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; + thr->jmp_bufs.PopBack(); + i--; + } + } } -TSAN_INTERCEPTOR(void, siglongjmp, void *env, int val) { - SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); - Printf("ThreadSanitizer: siglongjmp() is not supported\n"); - Die(); +static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { + if (thr->shadow_stack_pos == 0) // called from libc guts during bootstrap + return; + // Cleanup old bufs. + JmpBufGarbageCollect(thr, sp); + // Remember the buf. + JmpBuf *buf = thr->jmp_bufs.PushBack(); + buf->sp = sp; + buf->mangled_sp = mangled_sp; + buf->shadow_stack_pos = thr->shadow_stack_pos; +} + +static void LongJmp(ThreadState *thr, uptr *env) { + uptr mangled_sp = env[6]; + // Find the saved buf by mangled_sp. + for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { + JmpBuf *buf = &thr->jmp_bufs[i]; + if (buf->mangled_sp == mangled_sp) { + CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); + // Unwind the stack. + while (thr->shadow_stack_pos > buf->shadow_stack_pos) + FuncExit(thr); + JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp + return; + } + } + Printf("ThreadSanitizer: can't find longjmp buf\n"); + CHECK(0); +} + +// FIXME: put everything below into a common extern "C" block? +extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { + ScopedInRtl in_rtl; + SetJmp(cur_thread(), sp, mangled_sp); +} + +// Not called. Merely to satisfy TSAN_INTERCEPT(). +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor_setjmp(void *env); +extern "C" int __interceptor_setjmp(void *env) { + CHECK(0); + return 0; +} + +// FIXME: any reason to have a separate declaration? +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor__setjmp(void *env); +extern "C" int __interceptor__setjmp(void *env) { + CHECK(0); + return 0; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor_sigsetjmp(void *env); +extern "C" int __interceptor_sigsetjmp(void *env) { + CHECK(0); + return 0; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor___sigsetjmp(void *env); +extern "C" int __interceptor___sigsetjmp(void *env) { + CHECK(0); + return 0; +} + +extern "C" int setjmp(void *env); +extern "C" int _setjmp(void *env); +extern "C" int sigsetjmp(void *env); +extern "C" int __sigsetjmp(void *env); +DEFINE_REAL(int, setjmp, void *env) +DEFINE_REAL(int, _setjmp, void *env) +DEFINE_REAL(int, sigsetjmp, void *env) +DEFINE_REAL(int, __sigsetjmp, void *env) + +TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { + { + SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); + } + LongJmp(cur_thread(), env); + REAL(longjmp)(env, val); +} + +TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { + { + SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); + } + LongJmp(cur_thread(), env); + REAL(siglongjmp)(env, val); } TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (cur_thread()->in_symbolizer) + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) return __libc_malloc(size); void *p = 0; { @@ -355,21 +456,23 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (cur_thread()->in_symbolizer) + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) return __libc_calloc(size, n); - if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) return 0; + if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) + return AllocatorReturnNull(); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); p = user_alloc(thr, pc, n * size); - if (p) internal_memset(p, 0, n * size); + if (p) + internal_memset(p, 0, n * size); } invoke_malloc_hook(p, n * size); return p; } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (cur_thread()->in_symbolizer) + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) return __libc_realloc(p, size); if (p) invoke_free_hook(p); @@ -384,7 +487,7 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) return __libc_free(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -394,15 +497,22 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) return __libc_free(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); } +TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { + SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); + if (libjvm_check(pc)) + return malloc_usable_size(p); + return user_alloc_usable_size(thr, pc, p); +} + #define OPERATOR_NEW_BODY(mangled_name) \ - if (cur_thread()->in_symbolizer) \ + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) \ return __libc_malloc(size); \ void *p = 0; \ { \ @@ -412,36 +522,58 @@ TSAN_INTERCEPTOR(void, cfree, void *p) { invoke_malloc_hook(p, size); \ return p; +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size); void *operator new(__sanitizer::uptr size) { OPERATOR_NEW_BODY(_Znwm); } + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size); void *operator new[](__sanitizer::uptr size) { OPERATOR_NEW_BODY(_Znam); } + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::nothrow_t const&); void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t); } + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::nothrow_t const&); void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t); } #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ - if (cur_thread()->in_symbolizer) \ + if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) \ return __libc_free(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ user_free(thr, pc, ptr); +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr); void operator delete(void *ptr) { OPERATOR_DELETE_BODY(_ZdlPv); } + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr); void operator delete[](void *ptr) { OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); } + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const&); void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(_ZdaPv); } + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const&); void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); } @@ -479,30 +611,6 @@ TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { return res; } -TSAN_INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { - SCOPED_TSAN_INTERCEPTOR(strcmp, s1, s2); - uptr len = 0; - for (; s1[len] && s2[len]; len++) { - if (s1[len] != s2[len]) - break; - } - MemoryAccessRange(thr, pc, (uptr)s1, len + 1, false); - MemoryAccessRange(thr, pc, (uptr)s2, len + 1, false); - return s1[len] - s2[len]; -} - -TSAN_INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr n) { - SCOPED_TSAN_INTERCEPTOR(strncmp, s1, s2, n); - uptr len = 0; - for (; len < n && s1[len] && s2[len]; len++) { - if (s1[len] != s2[len]) - break; - } - MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); - MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); - return len == n ? 0 : s1[len] - s2[len]; -} - TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) { SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n); void *res = REAL(memchr)(s, c, n); @@ -572,6 +680,21 @@ TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) { return res; } +TSAN_INTERCEPTOR(char*, strdup, const char *str) { + SCOPED_TSAN_INTERCEPTOR(strdup, str); + if (libjvm_check(pc)) { + // The memory must come from libc malloc, + // and we must not instrument accesses in this case. + uptr n = internal_strlen(str) + 1; + void *p = __libc_malloc(n); + if (p == 0) + return 0; + return (char*)internal_memcpy(p, str, n); + } + // strdup will call malloc, so no instrumentation is required here. + return REAL(strdup)(str); +} + static bool fix_mmap_addr(void **addr, long_t sz, int flags) { if (*addr) { if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { @@ -595,7 +718,7 @@ TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); - MemoryResetRange(thr, pc, (uptr)res, sz); + MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); } return res; } @@ -609,13 +732,14 @@ TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); - MemoryResetRange(thr, pc, (uptr)res, sz); + MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); } return res; } TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); + DontNeedShadowFor((uptr)addr, sz); int res = REAL(munmap)(addr, sz); return res; } @@ -727,21 +851,21 @@ extern "C" void *__tsan_thread_start_func(void *arg) { TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_TSAN_INTERCEPTOR(pthread_create, th, attr, callback, param); - pthread_attr_t myattr; + __sanitizer_pthread_attr_t myattr; if (attr == 0) { pthread_attr_init(&myattr); attr = &myattr; } int detached = 0; pthread_attr_getdetachstate(attr, &detached); - uptr stacksize = 0; - pthread_attr_getstacksize(attr, &stacksize); - // We place the huge ThreadState object into TLS, account for that. - const uptr minstacksize = GetTlsSize() + 128*1024; - if (stacksize < minstacksize) { - DPrintf("ThreadSanitizer: stacksize %zu->%zu\n", stacksize, minstacksize); - pthread_attr_setstacksize(attr, minstacksize); - } + +#if defined(TSAN_DEBUG_OUTPUT) + int verbosity = (TSAN_DEBUG_OUTPUT); +#else + int verbosity = 0; +#endif + AdjustStackSizeLinux(attr, verbosity); + ThreadParam p; p.callback = callback; p.param = param; @@ -960,30 +1084,30 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { return res; } -// libpthread.so contains several versions of pthread_cond_init symbol. -// When we just dlsym() it, we get the wrong (old) version. -/* TSAN_INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, c, a); + MemoryWrite(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_init)(c, a); return res; } -*/ TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c); + MemoryWrite(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_destroy)(c); return res; } TSAN_INTERCEPTOR(int, pthread_cond_signal, void *c) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, c); + MemoryRead(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_signal)(c); return res; } TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, c); + MemoryRead(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_broadcast)(c); return res; } @@ -991,14 +1115,17 @@ TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) { TSAN_INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, c, m); MutexUnlock(thr, pc, (uptr)m); + MemoryRead(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_wait)(c, m); MutexLock(thr, pc, (uptr)m); return res; } -TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { +TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, + void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime); MutexUnlock(thr, pc, (uptr)m); + MemoryRead(thr, pc, (uptr)c, kSizeLog1); int res = REAL(pthread_cond_timedwait)(c, m, abstime); MutexLock(thr, pc, (uptr)m); return res; @@ -1309,22 +1436,6 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { return res; } -TSAN_INTERCEPTOR(int, accept, int fd, void *addr, unsigned *addrlen) { - SCOPED_TSAN_INTERCEPTOR(accept, fd, addr, addrlen); - int fd2 = REAL(accept)(fd, addr, addrlen); - if (fd >= 0 && fd2 >= 0) - FdSocketAccept(thr, pc, fd, fd2); - return fd2; -} - -TSAN_INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) { - SCOPED_TSAN_INTERCEPTOR(accept4, fd, addr, addrlen, f); - int fd2 = REAL(accept4)(fd, addr, addrlen, f); - if (fd >= 0 && fd2 >= 0) - FdSocketAccept(thr, pc, fd, fd2); - return fd2; -} - TSAN_INTERCEPTOR(int, epoll_create, int size) { SCOPED_TSAN_INTERCEPTOR(epoll_create, size); int fd = REAL(epoll_create)(size); @@ -1383,40 +1494,6 @@ TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { return res; } -TSAN_INTERCEPTOR(long_t, readv, int fd, void *vec, int cnt) { - SCOPED_TSAN_INTERCEPTOR(readv, fd, vec, cnt); - int res = REAL(readv)(fd, vec, cnt); - if (res >= 0 && fd >= 0) { - FdAcquire(thr, pc, fd); - } - return res; -} - -TSAN_INTERCEPTOR(long_t, preadv64, int fd, void *vec, int cnt, u64 off) { - SCOPED_TSAN_INTERCEPTOR(preadv64, fd, vec, cnt, off); - int res = REAL(preadv64)(fd, vec, cnt, off); - if (res >= 0 && fd >= 0) { - FdAcquire(thr, pc, fd); - } - return res; -} - -TSAN_INTERCEPTOR(long_t, writev, int fd, void *vec, int cnt) { - SCOPED_TSAN_INTERCEPTOR(writev, fd, vec, cnt); - if (fd >= 0) - FdRelease(thr, pc, fd); - int res = REAL(writev)(fd, vec, cnt); - return res; -} - -TSAN_INTERCEPTOR(long_t, pwritev64, int fd, void *vec, int cnt, u64 off) { - SCOPED_TSAN_INTERCEPTOR(pwritev64, fd, vec, cnt, off); - if (fd >= 0) - FdRelease(thr, pc, fd); - int res = REAL(pwritev64)(fd, vec, cnt, off); - return res; -} - TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); if (fd >= 0) @@ -1442,15 +1519,6 @@ TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { return res; } -TSAN_INTERCEPTOR(long_t, recvmsg, int fd, void *msg, int flags) { - SCOPED_TSAN_INTERCEPTOR(recvmsg, fd, msg, flags); - int res = REAL(recvmsg)(fd, msg, flags); - if (res >= 0 && fd >= 0) { - FdAcquire(thr, pc, fd); - } - return res; -} - TSAN_INTERCEPTOR(int, unlink, char *path) { SCOPED_TSAN_INTERCEPTOR(unlink, path); Release(thr, pc, File2addr(path)); @@ -1515,6 +1583,17 @@ TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { return REAL(fwrite)(p, size, nmemb, f); } +TSAN_INTERCEPTOR(int, fflush, void *stream) { + SCOPED_TSAN_INTERCEPTOR(fflush, stream); + return REAL(fflush)(stream); +} + +TSAN_INTERCEPTOR(void, abort, int fake) { + SCOPED_TSAN_INTERCEPTOR(abort, fake); + REAL(fflush)(0); + REAL(abort)(fake); +} + TSAN_INTERCEPTOR(int, puts, const char *s) { SCOPED_TSAN_INTERCEPTOR(puts, s); MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); @@ -1556,26 +1635,19 @@ TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { return res; } -TSAN_INTERCEPTOR(int, poll, void *fds, long_t nfds, int timeout) { - SCOPED_TSAN_INTERCEPTOR(poll, fds, nfds, timeout); - int res = BLOCK_REAL(poll)(fds, nfds, timeout); - return res; -} - -static void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, +void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, my_siginfo_t *info, void *ctx) { ThreadState *thr = cur_thread(); SignalContext *sctx = SigCtx(thr); // Don't mess with synchronous signals. if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || - sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || + sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || // If we are sending signal to ourselves, we must process it now. (sctx && sig == sctx->int_signal_send) || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, // i.e. pthread_join()->munmap()). (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) { - CHECK(thr->in_rtl == 0 || thr->in_rtl == 1); int in_rtl = thr->in_rtl; thr->in_rtl = 0; CHECK_EQ(thr->in_signal_handler, false); @@ -1621,7 +1693,7 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { internal_memcpy(&sigactions[sig], act, sizeof(*act)); sigaction_t newact; internal_memcpy(&newact, act, sizeof(newact)); - sigfillset(&newact.sa_mask); + REAL(sigfillset)(&newact.sa_mask); if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { if (newact.sa_flags & SA_SIGINFO) newact.sa_sigaction = rtl_sigaction; @@ -1644,6 +1716,11 @@ TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { return old.sa_handler; } +TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { + SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); + return REAL(sigsuspend)(mask); +} + TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); SignalContext *sctx = SigCtx(thr); @@ -1661,11 +1738,11 @@ TSAN_INTERCEPTOR(int, kill, int pid, int sig) { SignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; - if (pid == GetPid()) { + if (pid == (int)internal_getpid()) { sctx->int_signal_send = sig; } int res = REAL(kill)(pid, sig); - if (pid == GetPid()) { + if (pid == (int)internal_getpid()) { CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; } @@ -1694,13 +1771,30 @@ TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { return REAL(gettimeofday)(tv, tz); } +TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, + void *hints, void *rv) { + SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); + // We miss atomic synchronization in getaddrinfo, + // and can report false race between malloc and free + // inside of getaddrinfo. So ignore memory accesses. + ThreadIgnoreBegin(thr); + // getaddrinfo calls fopen, which can be intercepted by user. + thr->in_rtl--; + CHECK_EQ(thr->in_rtl, 0); + int res = REAL(getaddrinfo)(node, service, hints, rv); + thr->in_rtl++; + ThreadIgnoreEnd(thr); + return res; +} + // Linux kernel has a bug that leads to kernel deadlock if a process // maps TBs of memory and then calls mlock(). static void MlockIsUnsupported() { static atomic_uint8_t printed; if (atomic_exchange(&printed, 1, memory_order_relaxed)) return; - Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n"); + if (flags()->verbosity > 0) + Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n"); } TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) { @@ -1725,7 +1819,6 @@ TSAN_INTERCEPTOR(int, munlockall, void) { TSAN_INTERCEPTOR(int, fork, int fake) { SCOPED_TSAN_INTERCEPTOR(fork, fake); - // It's intercepted merely to process pending signals. int pid = REAL(fork)(fake); if (pid == 0) { // child @@ -1742,27 +1835,106 @@ struct TsanInterceptorContext { const uptr pc; }; -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - MemoryAccessRange(((TsanInterceptorContext*)ctx)->thr, \ - ((TsanInterceptorContext*)ctx)->pc, \ - (uptr)ptr, size, true) -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - MemoryAccessRange(((TsanInterceptorContext*)ctx)->thr, \ - ((TsanInterceptorContext*)ctx)->pc, \ - (uptr)ptr, size, false) -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__) \ - TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ - ctx = (void*)&_ctx; \ - (void)ctx; +#include "sanitizer_common/sanitizer_platform_interceptors.h" +// Causes interceptor recursion (getpwuid_r() calls fopen()) +#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS +#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS +// Causes interceptor recursion (getaddrinfo() and fopen()) +#undef SANITIZER_INTERCEPT_GETADDRINFO +#undef SANITIZER_INTERCEPT_GETNAMEINFO +// Causes interceptor recursion (glob64() calls lstat64()) +#undef SANITIZER_INTERCEPT_GLOB + +#define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ + true) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ + ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ + false) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ + TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ + ctx = (void *)&_ctx; \ + (void) ctx; #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - FdAcquire(((TsanInterceptorContext*)ctx)->thr, pc, fd) + FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - FdRelease(((TsanInterceptorContext*)ctx)->thr, pc, fd) + FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - ThreadSetName(((TsanInterceptorContext*)ctx)->thr, name) + ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) #include "sanitizer_common/sanitizer_common_interceptors.inc" +#define TSAN_SYSCALL() \ + ThreadState *thr = cur_thread(); \ + ScopedSyscall scoped_syscall(thr) \ +/**/ + +struct ScopedSyscall { + ThreadState *thr; + + explicit ScopedSyscall(ThreadState *thr) + : thr(thr) { + if (thr->in_rtl == 0) + Initialize(thr); + thr->in_rtl++; + } + + ~ScopedSyscall() { + thr->in_rtl--; + if (thr->in_rtl == 0) + ProcessPendingSignals(thr); + } +}; + +static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { + TSAN_SYSCALL(); + MemoryAccessRange(thr, pc, p, s, write); +} + +static void syscall_fd_close(uptr pc, int fd) { + TSAN_SYSCALL(); + if (fd >= 0) + FdClose(thr, pc, fd); +} + +static void syscall_pre_fork(uptr pc) { + TSAN_SYSCALL(); +} + +static void syscall_post_fork(uptr pc, int res) { + TSAN_SYSCALL(); + if (res == 0) { + // child + FdOnFork(thr, pc); + } else if (res > 0) { + // parent + } +} + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ + syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ + syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { } while (false) +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { } while (false) +#define COMMON_SYSCALL_FD_CLOSE(fd) \ + syscall_fd_close(GET_CALLER_PC(), fd) +#define COMMON_SYSCALL_PRE_FORK() \ + syscall_pre_fork(GET_CALLER_PC()) +#define COMMON_SYSCALL_POST_FORK(res) \ + syscall_post_fork(GET_CALLER_PC(), res) +#include "sanitizer_common/sanitizer_common_syscalls.inc" + namespace __tsan { void ProcessPendingSignals(ThreadState *thr) { @@ -1774,8 +1946,8 @@ void ProcessPendingSignals(ThreadState *thr) { thr->in_signal_handler = true; sctx->pending_signal_count = 0; // These are too big for stack. - static THREADLOCAL sigset_t emptyset, oldset; - sigfillset(&emptyset); + static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; + REAL(sigfillset)(&emptyset); pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); for (int sig = 0; sig < kSigCount; sig++) { SignalDesc *signal = &sctx->pending_signals[sig]; @@ -1796,8 +1968,9 @@ void ProcessPendingSignals(ThreadState *thr) { uptr pc = signal->sigaction ? (uptr)sigactions[sig].sa_sigaction : (uptr)sigactions[sig].sa_handler; + pc += 1; // return address is expected, OutputReport() will undo this stack.Init(&pc, 1); - Lock l(&ctx->thread_mtx); + ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); if (!IsFiredSuppression(ctx, rep, stack)) { rep.AddStack(&stack); @@ -1813,6 +1986,16 @@ void ProcessPendingSignals(ThreadState *thr) { thr->in_signal_handler = false; } +static void finalize(void *arg) { + ThreadState * thr = cur_thread(); + uptr pc = 0; + atexit_ctx->exit(thr, pc); + int status = Finalize(cur_thread()); + REAL(fflush)(0); + if (status) + _exit(status); +} + static void unreachable() { Printf("FATAL: ThreadSanitizer: unreachable called\n"); Die(); @@ -1826,8 +2009,16 @@ void InitializeInterceptors() { REAL(memcpy) = internal_memcpy; REAL(memcmp) = internal_memcmp; + // Instruct libc malloc to consume less memory. + mallopt(1, 0); // M_MXFAST + mallopt(-3, 32*1024); // M_MMAP_THRESHOLD + SANITIZER_COMMON_INTERCEPTORS_INIT; + TSAN_INTERCEPT(setjmp); + TSAN_INTERCEPT(_setjmp); + TSAN_INTERCEPT(sigsetjmp); + TSAN_INTERCEPT(__sigsetjmp); TSAN_INTERCEPT(longjmp); TSAN_INTERCEPT(siglongjmp); @@ -1848,7 +2039,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(strlen); TSAN_INTERCEPT(memset); TSAN_INTERCEPT(memcpy); - TSAN_INTERCEPT(strcmp); TSAN_INTERCEPT(memchr); TSAN_INTERCEPT(memrchr); TSAN_INTERCEPT(memmove); @@ -1856,10 +2046,10 @@ void InitializeInterceptors() { TSAN_INTERCEPT(strchr); TSAN_INTERCEPT(strchrnul); TSAN_INTERCEPT(strrchr); - TSAN_INTERCEPT(strncmp); TSAN_INTERCEPT(strcpy); // NOLINT TSAN_INTERCEPT(strncpy); TSAN_INTERCEPT(strstr); + TSAN_INTERCEPT(strdup); TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); @@ -1888,12 +2078,12 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_rwlock_timedwrlock); TSAN_INTERCEPT(pthread_rwlock_unlock); - // TSAN_INTERCEPT(pthread_cond_init); - TSAN_INTERCEPT(pthread_cond_destroy); - TSAN_INTERCEPT(pthread_cond_signal); - TSAN_INTERCEPT(pthread_cond_broadcast); - TSAN_INTERCEPT(pthread_cond_wait); - TSAN_INTERCEPT(pthread_cond_timedwait); + INTERCEPT_FUNCTION_VER(pthread_cond_init, GLIBC_2.3.2); + INTERCEPT_FUNCTION_VER(pthread_cond_destroy, GLIBC_2.3.2); + INTERCEPT_FUNCTION_VER(pthread_cond_signal, GLIBC_2.3.2); + INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, GLIBC_2.3.2); + INTERCEPT_FUNCTION_VER(pthread_cond_wait, GLIBC_2.3.2); + INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, GLIBC_2.3.2); TSAN_INTERCEPT(pthread_barrier_init); TSAN_INTERCEPT(pthread_barrier_destroy); @@ -1937,8 +2127,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(connect); TSAN_INTERCEPT(bind); TSAN_INTERCEPT(listen); - TSAN_INTERCEPT(accept); - TSAN_INTERCEPT(accept4); TSAN_INTERCEPT(epoll_create); TSAN_INTERCEPT(epoll_create1); TSAN_INTERCEPT(close); @@ -1947,14 +2135,9 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pipe); TSAN_INTERCEPT(pipe2); - TSAN_INTERCEPT(readv); - TSAN_INTERCEPT(preadv64); - TSAN_INTERCEPT(writev); - TSAN_INTERCEPT(pwritev64); TSAN_INTERCEPT(send); TSAN_INTERCEPT(sendmsg); TSAN_INTERCEPT(recv); - TSAN_INTERCEPT(recvmsg); TSAN_INTERCEPT(unlink); TSAN_INTERCEPT(fopen); @@ -1962,16 +2145,18 @@ void InitializeInterceptors() { TSAN_INTERCEPT(fclose); TSAN_INTERCEPT(fread); TSAN_INTERCEPT(fwrite); + TSAN_INTERCEPT(fflush); + TSAN_INTERCEPT(abort); TSAN_INTERCEPT(puts); TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(opendir); TSAN_INTERCEPT(epoll_ctl); TSAN_INTERCEPT(epoll_wait); - TSAN_INTERCEPT(poll); TSAN_INTERCEPT(sigaction); TSAN_INTERCEPT(signal); + TSAN_INTERCEPT(sigsuspend); TSAN_INTERCEPT(raise); TSAN_INTERCEPT(kill); TSAN_INTERCEPT(pthread_kill); @@ -1979,6 +2164,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(usleep); TSAN_INTERCEPT(nanosleep); TSAN_INTERCEPT(gettimeofday); + TSAN_INTERCEPT(getaddrinfo); TSAN_INTERCEPT(mlock); TSAN_INTERCEPT(munlock); diff --git a/libsanitizer/tsan/tsan_interface.cc b/libsanitizer/tsan/tsan_interface.cc index 992e3834aae..e056bd465bf 100644 --- a/libsanitizer/tsan/tsan_interface.cc +++ b/libsanitizer/tsan/tsan_interface.cc @@ -12,11 +12,16 @@ #include "tsan_interface.h" #include "tsan_interface_ann.h" #include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #define CALLERPC ((uptr)__builtin_return_address(0)) using namespace __tsan; // NOLINT +typedef u16 uint16_t; +typedef u32 uint32_t; +typedef u64 uint64_t; + void __tsan_init() { Initialize(cur_thread()); } @@ -31,6 +36,57 @@ void __tsan_write16(void *addr) { MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8); } +u16 __tsan_unaligned_read2(const uu16 *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false); + return *addr; +} + +u32 __tsan_unaligned_read4(const uu32 *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false); + return *addr; +} + +u64 __tsan_unaligned_read8(const uu64 *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false); + return *addr; +} + +void __tsan_unaligned_write2(uu16 *addr, u16 v) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false); + *addr = v; +} + +void __tsan_unaligned_write4(uu32 *addr, u32 v) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false); + *addr = v; +} + +void __tsan_unaligned_write8(uu64 *addr, u64 v) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false); + *addr = v; +} + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +uint16_t __sanitizer_unaligned_load16(void *addr) + ALIAS("__tsan_unaligned_read2"); +SANITIZER_INTERFACE_ATTRIBUTE +uint32_t __sanitizer_unaligned_load32(void *addr) + ALIAS("__tsan_unaligned_read4"); +SANITIZER_INTERFACE_ATTRIBUTE +uint64_t __sanitizer_unaligned_load64(void *addr) + ALIAS("__tsan_unaligned_read8"); +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store16(void *addr, uint16_t v) + ALIAS("__tsan_unaligned_write2"); +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store32(void *addr, uint32_t v) + ALIAS("__tsan_unaligned_write4"); +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store64(void *addr, uint64_t v) + ALIAS("__tsan_unaligned_write8"); +} + void __tsan_acquire(void *addr) { Acquire(cur_thread(), CALLERPC, (uptr)addr); } diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h index 2cfd7684183..63de7b2ab5c 100644 --- a/libsanitizer/tsan/tsan_interface.h +++ b/libsanitizer/tsan/tsan_interface.h @@ -25,30 +25,38 @@ extern "C" { // This function should be called at the very beginning of the process, // before any instrumented code is executed and before any call to malloc. -void __tsan_init() SANITIZER_INTERFACE_ATTRIBUTE; - -void __tsan_read1(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_read2(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_read4(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_read8(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_read16(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; - -void __tsan_write1(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_write2(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_write4(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_write8(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_write16(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; - -void __tsan_vptr_update(void **vptr_p, void *new_val) - SANITIZER_INTERFACE_ATTRIBUTE; - -void __tsan_func_entry(void *call_pc) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_func_exit() SANITIZER_INTERFACE_ATTRIBUTE; - -void __tsan_read_range(void *addr, unsigned long size) // NOLINT - SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_write_range(void *addr, unsigned long size) // NOLINT - SANITIZER_INTERFACE_ATTRIBUTE; +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init(); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16(void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __tsan_unaligned_read2(const uu16 *addr); +SANITIZER_INTERFACE_ATTRIBUTE u32 __tsan_unaligned_read4(const uu32 *addr); +SANITIZER_INTERFACE_ATTRIBUTE u64 __tsan_unaligned_read8(const uu64 *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(uu16 *addr, u16 v); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(uu32 *addr, u32 v); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(uu64 *addr, u64 v); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_vptr_update(void **vptr_p, void *new_val); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_read_range(void *addr, unsigned long size); // NOLINT +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_write_range(void *addr, unsigned long size); // NOLINT #ifdef __cplusplus } // extern "C" diff --git a/libsanitizer/tsan/tsan_interface_ann.cc b/libsanitizer/tsan/tsan_interface_ann.cc index 7e49fb8b059..46d9e3ec1af 100644 --- a/libsanitizer/tsan/tsan_interface_ann.cc +++ b/libsanitizer/tsan/tsan_interface_ann.cc @@ -11,6 +11,7 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stacktrace.h" #include "tsan_interface_ann.h" #include "tsan_mutex.h" #include "tsan_report.h" @@ -18,6 +19,7 @@ #include "tsan_mman.h" #include "tsan_flags.h" #include "tsan_platform.h" +#include "tsan_vector.h" #define CALLERPC ((uptr)__builtin_return_address(0)) @@ -65,6 +67,7 @@ struct ExpectRace { ExpectRace *next; ExpectRace *prev; int hitcount; + int addcount; uptr addr; uptr size; char *file; @@ -89,16 +92,19 @@ static void AddExpectRace(ExpectRace *list, char *f, int l, uptr addr, uptr size, char *desc) { ExpectRace *race = list->next; for (; race != list; race = race->next) { - if (race->addr == addr && race->size == size) + if (race->addr == addr && race->size == size) { + race->addcount++; return; + } } race = (ExpectRace*)internal_alloc(MBlockExpectRace, sizeof(ExpectRace)); - race->hitcount = 0; race->addr = addr; race->size = size; race->file = f; race->line = l; race->desc[0] = 0; + race->hitcount = 0; + race->addcount = 1; if (desc) { int i = 0; for (; i < kMaxDescLen - 1 && desc[i]; i++) @@ -153,6 +159,68 @@ bool IsExpectedReport(uptr addr, uptr size) { return false; } +static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched, + int *unique_count, int *hit_count, int ExpectRace::*counter) { + ExpectRace *list = &dyn_ann_ctx->benign; + for (ExpectRace *race = list->next; race != list; race = race->next) { + (*unique_count)++; + if (race->*counter == 0) + continue; + (*hit_count) += race->*counter; + uptr i = 0; + for (; i < matched->Size(); i++) { + ExpectRace *race0 = &(*matched)[i]; + if (race->line == race0->line + && internal_strcmp(race->file, race0->file) == 0 + && internal_strcmp(race->desc, race0->desc) == 0) { + race0->*counter += race->*counter; + break; + } + } + if (i == matched->Size()) + matched->PushBack(*race); + } +} + +void PrintMatchedBenignRaces() { + Lock lock(&dyn_ann_ctx->mtx); + int unique_count = 0; + int hit_count = 0; + int add_count = 0; + Vector<ExpectRace> hit_matched(MBlockScopedBuf); + CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count, + &ExpectRace::hitcount); + Vector<ExpectRace> add_matched(MBlockScopedBuf); + CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count, + &ExpectRace::addcount); + if (hit_matched.Size()) { + Printf("ThreadSanitizer: Matched %d \"benign\" races (pid=%d):\n", + hit_count, (int)internal_getpid()); + for (uptr i = 0; i < hit_matched.Size(); i++) { + Printf("%d %s:%d %s\n", + hit_matched[i].hitcount, hit_matched[i].file, + hit_matched[i].line, hit_matched[i].desc); + } + } + if (hit_matched.Size()) { + Printf("ThreadSanitizer: Annotated %d \"benign\" races, %d unique" + " (pid=%d):\n", + add_count, unique_count, (int)internal_getpid()); + for (uptr i = 0; i < add_matched.Size(); i++) { + Printf("%d %s:%d %s\n", + add_matched[i].addcount, add_matched[i].file, + add_matched[i].line, add_matched[i].desc); + } + } +} + +static void ReportMissedExpectedRace(ExpectRace *race) { + Printf("==================\n"); + Printf("WARNING: ThreadSanitizer: missed expected data race\n"); + Printf(" %s addr=%zx %s:%d\n", + race->desc, race->addr, race->file, race->line); + Printf("==================\n"); +} } // namespace __tsan using namespace __tsan; // NOLINT @@ -160,12 +228,12 @@ using namespace __tsan; // NOLINT extern "C" { void INTERFACE_ATTRIBUTE AnnotateHappensBefore(char *f, int l, uptr addr) { SCOPED_ANNOTATION(AnnotateHappensBefore); - Release(cur_thread(), CALLERPC, addr); + Release(thr, pc, addr); } void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) { SCOPED_ANNOTATION(AnnotateHappensAfter); - Acquire(cur_thread(), CALLERPC, addr); + Acquire(thr, pc, addr); } void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) { @@ -235,14 +303,6 @@ void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) { SCOPED_ANNOTATION(AnnotateNoOp); } -static void ReportMissedExpectedRace(ExpectRace *race) { - Printf("==================\n"); - Printf("WARNING: ThreadSanitizer: missed expected data race\n"); - Printf(" %s addr=%zx %s:%d\n", - race->desc, race->addr, race->file, race->line); - Printf("==================\n"); -} - void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) { SCOPED_ANNOTATION(AnnotateFlushExpectedRaces); Lock lock(&dyn_ann_ctx->mtx); @@ -321,22 +381,22 @@ void INTERFACE_ATTRIBUTE AnnotateBenignRace( void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) { SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin); - IgnoreCtl(cur_thread(), false, true); + ThreadIgnoreBegin(thr); } void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) { SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd); - IgnoreCtl(cur_thread(), false, false); + ThreadIgnoreEnd(thr); } void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) { SCOPED_ANNOTATION(AnnotateIgnoreWritesBegin); - IgnoreCtl(cur_thread(), true, true); + ThreadIgnoreBegin(thr); } void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) { SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd); - IgnoreCtl(thr, true, false); + ThreadIgnoreEnd(thr); } void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange( @@ -355,6 +415,9 @@ void INTERFACE_ATTRIBUTE AnnotateThreadName( ThreadSetName(thr, name); } +// We deliberately omit the implementation of WTFAnnotateHappensBefore() and +// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate +// atomic operations, which should be handled by ThreadSanitizer correctly. void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) { SCOPED_ANNOTATION(AnnotateHappensBefore); } @@ -366,6 +429,7 @@ void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) { void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized( char *f, int l, uptr mem, uptr sz, char *desc) { SCOPED_ANNOTATION(AnnotateBenignRaceSized); + BenignRaceImpl(f, l, mem, 1, desc); } int INTERFACE_ATTRIBUTE RunningOnValgrind() { @@ -382,4 +446,7 @@ const char INTERFACE_ATTRIBUTE* ThreadSanitizerQuery(const char *query) { else return "0"; } + +void INTERFACE_ATTRIBUTE +AnnotateMemoryIsInitialized(char *f, int l, uptr mem, uptr sz) {} } // extern "C" diff --git a/libsanitizer/tsan/tsan_interface_ann.h b/libsanitizer/tsan/tsan_interface_ann.h index b6500329428..45c18352e69 100644 --- a/libsanitizer/tsan/tsan_interface_ann.h +++ b/libsanitizer/tsan/tsan_interface_ann.h @@ -21,8 +21,8 @@ extern "C" { #endif -void __tsan_acquire(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; -void __tsan_release(void *addr) SANITIZER_INTERFACE_ATTRIBUTE; +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_acquire(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_release(void *addr); #ifdef __cplusplus } // extern "C" diff --git a/libsanitizer/tsan/tsan_interface_atomic.cc b/libsanitizer/tsan/tsan_interface_atomic.cc index 2c8b2ab049c..02ebb47e6af 100644 --- a/libsanitizer/tsan/tsan_interface_atomic.cc +++ b/libsanitizer/tsan/tsan_interface_atomic.cc @@ -28,23 +28,37 @@ using namespace __tsan; // NOLINT #define SCOPED_ATOMIC(func, ...) \ const uptr callpc = (uptr)__builtin_return_address(0); \ uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ - pc = __sanitizer::StackTrace::GetPreviousInstructionPc(pc); \ mo = ConvertOrder(mo); \ mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ ThreadState *const thr = cur_thread(); \ AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ - ScopedAtomic sa(thr, callpc, __FUNCTION__); \ + ScopedAtomic sa(thr, callpc, a, mo, __FUNCTION__); \ return Atomic##func(thr, pc, __VA_ARGS__); \ /**/ +// Some shortcuts. +typedef __tsan_memory_order morder; +typedef __tsan_atomic8 a8; +typedef __tsan_atomic16 a16; +typedef __tsan_atomic32 a32; +typedef __tsan_atomic64 a64; +typedef __tsan_atomic128 a128; +const morder mo_relaxed = __tsan_memory_order_relaxed; +const morder mo_consume = __tsan_memory_order_consume; +const morder mo_acquire = __tsan_memory_order_acquire; +const morder mo_release = __tsan_memory_order_release; +const morder mo_acq_rel = __tsan_memory_order_acq_rel; +const morder mo_seq_cst = __tsan_memory_order_seq_cst; + class ScopedAtomic { public: - ScopedAtomic(ThreadState *thr, uptr pc, const char *func) + ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, + morder mo, const char *func) : thr_(thr) { CHECK_EQ(thr_->in_rtl, 0); ProcessPendingSignals(thr); FuncEntry(thr_, pc); - DPrintf("#%d: %s\n", thr_->tid, func); + DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); thr_->in_rtl++; } ~ScopedAtomic() { @@ -56,20 +70,6 @@ class ScopedAtomic { ThreadState *thr_; }; -// Some shortcuts. -typedef __tsan_memory_order morder; -typedef __tsan_atomic8 a8; -typedef __tsan_atomic16 a16; -typedef __tsan_atomic32 a32; -typedef __tsan_atomic64 a64; -typedef __tsan_atomic128 a128; -const morder mo_relaxed = __tsan_memory_order_relaxed; -const morder mo_consume = __tsan_memory_order_consume; -const morder mo_acquire = __tsan_memory_order_acquire; -const morder mo_release = __tsan_memory_order_release; -const morder mo_acq_rel = __tsan_memory_order_acq_rel; -const morder mo_seq_cst = __tsan_memory_order_seq_cst; - static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { StatInc(thr, StatAtomic); StatInc(thr, t); @@ -288,16 +288,20 @@ static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, template<typename T, T (*F)(volatile T *v, T op)> static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); - thr->clock.set(thr->tid, thr->fast_state.epoch()); - if (IsAcqRelOrder(mo)) - thr->clock.acq_rel(&s->clock); - else if (IsReleaseOrder(mo)) - thr->clock.release(&s->clock); - else if (IsAcquireOrder(mo)) - thr->clock.acquire(&s->clock); + SyncVar *s = 0; + if (mo != mo_relaxed) { + s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); + thr->clock.set(thr->tid, thr->fast_state.epoch()); + if (IsAcqRelOrder(mo)) + thr->clock.acq_rel(&s->clock); + else if (IsReleaseOrder(mo)) + thr->clock.release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.acquire(&s->clock); + } v = F(a, v); - s->mtx.Unlock(); + if (s) + s->mtx.Unlock(); return v; } @@ -348,17 +352,21 @@ static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo, morder fmo) { (void)fmo; // Unused because llvm does not pass it yet. MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); - thr->clock.set(thr->tid, thr->fast_state.epoch()); - if (IsAcqRelOrder(mo)) - thr->clock.acq_rel(&s->clock); - else if (IsReleaseOrder(mo)) - thr->clock.release(&s->clock); - else if (IsAcquireOrder(mo)) - thr->clock.acquire(&s->clock); + SyncVar *s = 0; + if (mo != mo_relaxed) { + s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); + thr->clock.set(thr->tid, thr->fast_state.epoch()); + if (IsAcqRelOrder(mo)) + thr->clock.acq_rel(&s->clock); + else if (IsReleaseOrder(mo)) + thr->clock.release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.acquire(&s->clock); + } T cc = *c; T pr = func_cas(a, cc, v); - s->mtx.Unlock(); + if (s) + s->mtx.Unlock(); if (pr == cc) return true; *c = pr; @@ -649,14 +657,14 @@ a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, } #if __TSAN_HAS_INT128 -a128 __tsan_atomic64_compare_exchange_val(volatile a128 *a, a128 c, a128 v, +a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif void __tsan_atomic_thread_fence(morder mo) { - char* a; + char* a = 0; SCOPED_ATOMIC(Fence, mo); } diff --git a/libsanitizer/tsan/tsan_interface_inl.h b/libsanitizer/tsan/tsan_interface_inl.h index 92796d1178f..4d6e10bcf7b 100644 --- a/libsanitizer/tsan/tsan_interface_inl.h +++ b/libsanitizer/tsan/tsan_interface_inl.h @@ -50,8 +50,20 @@ void __tsan_write8(void *addr) { void __tsan_vptr_update(void **vptr_p, void *new_val) { CHECK_EQ(sizeof(vptr_p), 8); - if (*vptr_p != new_val) - MemoryWrite(cur_thread(), CALLERPC, (uptr)vptr_p, kSizeLog8); + if (*vptr_p != new_val) { + ThreadState *thr = cur_thread(); + thr->is_vptr_access = true; + MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); + thr->is_vptr_access = false; + } +} + +void __tsan_vptr_read(void **vptr_p) { + CHECK_EQ(sizeof(vptr_p), 8); + ThreadState *thr = cur_thread(); + thr->is_vptr_access = true; + MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); + thr->is_vptr_access = false; } void __tsan_func_entry(void *pc) { diff --git a/libsanitizer/tsan/tsan_interface_java.cc b/libsanitizer/tsan/tsan_interface_java.cc index f8c0b4eb635..7cc72577977 100644 --- a/libsanitizer/tsan/tsan_interface_java.cc +++ b/libsanitizer/tsan/tsan_interface_java.cc @@ -15,6 +15,8 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_procmaps.h" using namespace __tsan; // NOLINT @@ -92,6 +94,8 @@ class ScopedJavaFunc { static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1]; static JavaContext *jctx; +extern atomic_uintptr_t libjvm_begin; +extern atomic_uintptr_t libjvm_end; static BlockDesc *getblock(uptr addr) { uptr i = (addr - jctx->heap_begin) / kHeapAlignment; @@ -155,11 +159,22 @@ SyncVar* GetAndRemoveJavaSync(ThreadState *thr, uptr pc, uptr addr) { #define SCOPED_JAVA_FUNC(func) \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ - const uptr pc = (uptr)&func; \ + const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ (void)pc; \ ScopedJavaFunc scoped(thr, caller_pc); \ /**/ +void __tsan_java_preinit(const char *libjvm_path) { + SCOPED_JAVA_FUNC(__tsan_java_preinit); + if (libjvm_path) { + uptr begin, end; + if (GetCodeRangeForFile(libjvm_path, &begin, &end)) { + atomic_store(&libjvm_begin, begin, memory_order_relaxed); + atomic_store(&libjvm_end, end, memory_order_relaxed); + } + } +} + void __tsan_java_init(jptr heap_begin, jptr heap_size) { SCOPED_JAVA_FUNC(__tsan_java_init); DPrintf("#%d: java_init(%p, %p)\n", thr->tid, heap_begin, heap_size); @@ -269,6 +284,7 @@ void __tsan_java_mutex_lock(jptr addr) { CHECK_GE(addr, jctx->heap_begin); CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + MutexCreate(thr, pc, addr, true, true, true); MutexLock(thr, pc, addr); } @@ -289,6 +305,7 @@ void __tsan_java_mutex_read_lock(jptr addr) { CHECK_GE(addr, jctx->heap_begin); CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + MutexCreate(thr, pc, addr, true, true, true); MutexReadLock(thr, pc, addr); } @@ -301,3 +318,25 @@ void __tsan_java_mutex_read_unlock(jptr addr) { MutexReadUnlock(thr, pc, addr); } + +void __tsan_java_mutex_lock_rec(jptr addr, int rec) { + SCOPED_JAVA_FUNC(__tsan_java_mutex_lock_rec); + DPrintf("#%d: java_mutex_lock_rec(%p, %d)\n", thr->tid, addr, rec); + CHECK_NE(jctx, 0); + CHECK_GE(addr, jctx->heap_begin); + CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + CHECK_GT(rec, 0); + + MutexCreate(thr, pc, addr, true, true, true); + MutexLock(thr, pc, addr, rec); +} + +int __tsan_java_mutex_unlock_rec(jptr addr) { + SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock_rec); + DPrintf("#%d: java_mutex_unlock_rec(%p)\n", thr->tid, addr); + CHECK_NE(jctx, 0); + CHECK_GE(addr, jctx->heap_begin); + CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + return MutexUnlock(thr, pc, addr, true); +} diff --git a/libsanitizer/tsan/tsan_interface_java.h b/libsanitizer/tsan/tsan_interface_java.h index 01922bc9231..818c07b97d4 100644 --- a/libsanitizer/tsan/tsan_interface_java.h +++ b/libsanitizer/tsan/tsan_interface_java.h @@ -32,7 +32,11 @@ extern "C" { typedef unsigned long jptr; // NOLINT -// Must be called before any other callback from Java. +// Must be called before any other callback from Java, right after dlopen +// of JVM shared lib. If libjvm_path is specified, then all interceptors +// coming directly from JVM will be ignored. +void __tsan_java_preinit(const char *libjvm_path) INTERFACE_ATTRIBUTE; +// Must be called after __tsan_java_preinit but before any other callback. void __tsan_java_init(jptr heap_begin, jptr heap_size) INTERFACE_ATTRIBUTE; // Must be called when the application exits. // Not necessary the last callback (concurrently running threads are OK). @@ -53,8 +57,7 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) INTERFACE_ATTRIBUTE; // Mutex lock. // Addr is any unique address associated with the mutex. -// Must not be called on recursive reentry. -// Object.wait() is handled as a pair of unlock/lock. +// Can be called on recursive reentry. void __tsan_java_mutex_lock(jptr addr) INTERFACE_ATTRIBUTE; // Mutex unlock. void __tsan_java_mutex_unlock(jptr addr) INTERFACE_ATTRIBUTE; @@ -62,6 +65,16 @@ void __tsan_java_mutex_unlock(jptr addr) INTERFACE_ATTRIBUTE; void __tsan_java_mutex_read_lock(jptr addr) INTERFACE_ATTRIBUTE; // Mutex read unlock. void __tsan_java_mutex_read_unlock(jptr addr) INTERFACE_ATTRIBUTE; +// Recursive mutex lock, intended for handling of Object.wait(). +// The 'rec' value must be obtained from the previous +// __tsan_java_mutex_unlock_rec(). +void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE; +// Recursive mutex unlock, intended for handling of Object.wait(). +// The return value says how many times this thread called lock() +// w/o a pairing unlock() (i.e. how many recursive levels it unlocked). +// It must be passed back to __tsan_java_mutex_lock_rec() to restore +// the same recursion level. +int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE; #ifdef __cplusplus } // extern "C" diff --git a/libsanitizer/tsan/tsan_mman.cc b/libsanitizer/tsan/tsan_mman.cc index 23c73c50cc1..832374becf5 100644 --- a/libsanitizer/tsan/tsan_mman.cc +++ b/libsanitizer/tsan/tsan_mman.cc @@ -27,6 +27,41 @@ extern "C" void WEAK __tsan_free_hook(void *ptr) { namespace __tsan { +COMPILER_CHECK(sizeof(MBlock) == 16); + +void MBlock::Lock() { + atomic_uintptr_t *a = reinterpret_cast<atomic_uintptr_t*>(this); + uptr v = atomic_load(a, memory_order_relaxed); + for (int iter = 0;; iter++) { + if (v & 1) { + if (iter < 10) + proc_yield(20); + else + internal_sched_yield(); + v = atomic_load(a, memory_order_relaxed); + continue; + } + if (atomic_compare_exchange_weak(a, &v, v | 1, memory_order_acquire)) + break; + } +} + +void MBlock::Unlock() { + atomic_uintptr_t *a = reinterpret_cast<atomic_uintptr_t*>(this); + uptr v = atomic_load(a, memory_order_relaxed); + DCHECK(v & 1); + atomic_store(a, v & ~1, memory_order_relaxed); +} + +struct MapUnmapCallback { + void OnMap(uptr p, uptr size) const { } + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + DontNeedShadowFor(p, size); + } +}; + static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64); Allocator *allocator() { return reinterpret_cast<Allocator*>(&allocator_placeholder); @@ -38,10 +73,12 @@ void InitializeAllocator() { void AllocatorThreadStart(ThreadState *thr) { allocator()->InitCache(&thr->alloc_cache); + internal_allocator()->InitCache(&thr->internal_alloc_cache); } void AllocatorThreadFinish(ThreadState *thr) { allocator()->DestroyCache(&thr->alloc_cache); + internal_allocator()->DestroyCache(&thr->internal_alloc_cache); } void AllocatorPrintStats() { @@ -54,7 +91,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { Context *ctx = CTX(); StackTrace stack; stack.ObtainCurrent(thr, pc); - Lock l(&ctx->thread_mtx); + ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeSignalUnsafe); if (!IsFiredSuppression(ctx, rep, stack)) { rep.AddStack(&stack); @@ -64,16 +101,18 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) { CHECK_GT(thr->in_rtl, 0); + if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) + return AllocatorReturnNull(); void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); if (p == 0) return 0; MBlock *b = new(allocator()->GetMetaData(p)) MBlock; - b->size = sz; - b->head = 0; - b->alloc_tid = thr->unique_id; - b->alloc_stack_id = CurrentStackId(thr, pc); + b->Init(sz, thr->tid, CurrentStackId(thr, pc)); if (CTX() && CTX()->initialized) { - MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); + if (thr->ignore_reads_and_writes == 0) + MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); + else + MemoryResetRange(thr, pc, (uptr)p, sz); } DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); SignalUnsafeCall(thr, pc); @@ -85,9 +124,9 @@ void user_free(ThreadState *thr, uptr pc, void *p) { CHECK_NE(p, (void*)0); DPrintf("#%d: free(%p)\n", thr->tid, p); MBlock *b = (MBlock*)allocator()->GetMetaData(p); - if (b->head) { - Lock l(&b->mtx); - for (SyncVar *s = b->head; s;) { + if (b->ListHead()) { + MBlock::ScopedLock l(b); + for (SyncVar *s = b->ListHead(); s;) { SyncVar *res = s; s = s->next; StatInc(thr, StatSyncDestroyed); @@ -95,12 +134,12 @@ void user_free(ThreadState *thr, uptr pc, void *p) { res->mtx.Unlock(); DestroyAndFree(res); } - b->head = 0; + b->ListReset(); } if (CTX() && CTX()->initialized && thr->in_rtl == 1) { - MemoryRangeFreed(thr, pc, (uptr)p, b->size); + if (thr->ignore_reads_and_writes == 0) + MemoryRangeFreed(thr, pc, (uptr)p, b->Size()); } - b->~MBlock(); allocator()->Deallocate(&thr->alloc_cache, p); SignalUnsafeCall(thr, pc); } @@ -116,20 +155,29 @@ void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { return 0; if (p) { MBlock *b = user_mblock(thr, p); - internal_memcpy(p2, p, min(b->size, sz)); + CHECK_NE(b, 0); + internal_memcpy(p2, p, min(b->Size(), sz)); } } - if (p) { + if (p) user_free(thr, pc, p); - } return p2; } +uptr user_alloc_usable_size(ThreadState *thr, uptr pc, void *p) { + CHECK_GT(thr->in_rtl, 0); + if (p == 0) + return 0; + MBlock *b = (MBlock*)allocator()->GetMetaData(p); + return b ? b->Size() : 0; +} + MBlock *user_mblock(ThreadState *thr, void *p) { - CHECK_NE(p, (void*)0); + CHECK_NE(p, 0); Allocator *a = allocator(); void *b = a->GetBlockBegin(p); - CHECK_NE(b, 0); + if (b == 0) + return 0; return (MBlock*)a->GetMetaData(b); } @@ -152,11 +200,12 @@ void invoke_free_hook(void *ptr) { void *internal_alloc(MBlockType typ, uptr sz) { ThreadState *thr = cur_thread(); CHECK_GT(thr->in_rtl, 0); + CHECK_LE(sz, InternalSizeClassMap::kMaxSize); if (thr->nomalloc) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } - return InternalAlloc(sz); + return InternalAlloc(sz, &thr->internal_alloc_cache); } void internal_free(void *p) { @@ -166,7 +215,7 @@ void internal_free(void *p) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } - InternalFree(p); + InternalFree(p, &thr->internal_alloc_cache); } } // namespace __tsan @@ -213,6 +262,12 @@ uptr __tsan_get_allocated_size(void *p) { if (p == 0) return 0; MBlock *b = (MBlock*)allocator()->GetMetaData(p); - return b->size; + return b->Size(); +} + +void __tsan_on_thread_idle() { + ThreadState *thr = cur_thread(); + allocator()->SwallowCache(&thr->alloc_cache); + internal_allocator()->SwallowCache(&thr->internal_alloc_cache); } } // extern "C" diff --git a/libsanitizer/tsan/tsan_mman.h b/libsanitizer/tsan/tsan_mman.h index 7a657e124bf..90faaffa1fc 100644 --- a/libsanitizer/tsan/tsan_mman.h +++ b/libsanitizer/tsan/tsan_mman.h @@ -29,6 +29,7 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, void user_free(ThreadState *thr, uptr pc, void *p); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align); +uptr user_alloc_usable_size(ThreadState *thr, uptr pc, void *p); // Given the pointer p into a valid allocated block, // returns the descriptor of the block. MBlock *user_mblock(ThreadState *thr, void *p); @@ -60,6 +61,7 @@ enum MBlockType { MBlockExpectRace, MBlockSignal, MBlockFD, + MBlockJmpBuf, // This must be the last. MBlockTypeCount diff --git a/libsanitizer/tsan/tsan_mutex.cc b/libsanitizer/tsan/tsan_mutex.cc index 716722b0897..e7846c53e4a 100644 --- a/libsanitizer/tsan/tsan_mutex.cc +++ b/libsanitizer/tsan/tsan_mutex.cc @@ -29,8 +29,8 @@ static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { /*0 MutexTypeInvalid*/ {}, /*1 MutexTypeTrace*/ {MutexTypeLeaf}, /*2 MutexTypeThreads*/ {MutexTypeReport}, - /*3 MutexTypeReport*/ {MutexTypeSyncTab, MutexTypeMBlock, - MutexTypeJavaMBlock}, + /*3 MutexTypeReport*/ {MutexTypeSyncTab, MutexTypeSyncVar, + MutexTypeMBlock, MutexTypeJavaMBlock}, /*4 MutexTypeSyncVar*/ {}, /*5 MutexTypeSyncTab*/ {MutexTypeSyncVar}, /*6 MutexTypeSlab*/ {MutexTypeLeaf}, diff --git a/libsanitizer/tsan/tsan_mutexset.h b/libsanitizer/tsan/tsan_mutexset.h index 6924eade4c6..ef60bd47db2 100644 --- a/libsanitizer/tsan/tsan_mutexset.h +++ b/libsanitizer/tsan/tsan_mutexset.h @@ -20,7 +20,7 @@ class MutexSet { public: // Holds limited number of mutexes. // The oldest mutexes are discarded on overflow. - static const uptr kMaxSize = 64; + static const uptr kMaxSize = 16; struct Desc { u64 id; u64 epoch; diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index 78c1a7d8195..ac36c5ab67a 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -35,9 +35,9 @@ C++ COMPAT linux memory layout: Go linux and darwin memory layout: 0000 0000 0000 - 0000 1000 0000: executable 0000 1000 0000 - 00f8 0000 0000: - -00f8 0000 0000 - 0118 0000 0000: heap -0118 0000 0000 - 1000 0000 0000: - -1000 0000 0000 - 1460 0000 0000: shadow +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 1380 0000 0000: shadow 1460 0000 0000 - 6000 0000 0000: - 6000 0000 0000 - 6200 0000 0000: traces 6200 0000 0000 - 7fff ffff ffff: - @@ -45,8 +45,8 @@ Go linux and darwin memory layout: Go windows memory layout: 0000 0000 0000 - 0000 1000 0000: executable 0000 1000 0000 - 00f8 0000 0000: - -00f8 0000 0000 - 0118 0000 0000: heap -0118 0000 0000 - 0100 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 0100 0000 0000: - 0100 0000 0000 - 0560 0000 0000: shadow 0560 0000 0000 - 0760 0000 0000: traces 0760 0000 0000 - 07ff ffff ffff: - @@ -63,11 +63,11 @@ namespace __tsan { #if defined(TSAN_GO) static const uptr kLinuxAppMemBeg = 0x000000000000ULL; -static const uptr kLinuxAppMemEnd = 0x00fcffffffffULL; -# if defined(_WIN32) +static const uptr kLinuxAppMemEnd = 0x04dfffffffffULL; +# if SANITIZER_WINDOWS static const uptr kLinuxShadowMsk = 0x010000000000ULL; # else -static const uptr kLinuxShadowMsk = 0x100000000000ULL; +static const uptr kLinuxShadowMsk = 0x200000000000ULL; # endif // TSAN_COMPAT_SHADOW is intended for COMPAT virtual memory layout, // when memory addresses are of the 0x2axxxxxxxxxx form. @@ -82,7 +82,7 @@ static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL; static const uptr kLinuxAppMemMsk = 0x7c0000000000ULL; -#if defined(_WIN32) +#if SANITIZER_WINDOWS const uptr kTraceMemBegin = 0x056000000000ULL; #else const uptr kTraceMemBegin = 0x600000000000ULL; @@ -130,13 +130,19 @@ static inline uptr AlternativeAddress(uptr addr) { #endif } -uptr GetShadowMemoryConsumption(); void FlushShadowMemory(); +void WriteMemoryProfile(char *buf, uptr buf_size); const char *InitializePlatform(); void FinalizePlatform(); -uptr ALWAYS_INLINE INLINE GetThreadTrace(int tid) { - uptr p = kTraceMemBegin + (uptr)tid * kTraceSize * sizeof(Event); +uptr ALWAYS_INLINE GetThreadTrace(int tid) { + uptr p = kTraceMemBegin + (uptr)(tid * 2) * kTraceSize * sizeof(Event); + DCHECK_LT(p, kTraceMemBegin + kTraceMemSize); + return p; +} + +uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) { + uptr p = kTraceMemBegin + (uptr)(tid * 2 + 1) * kTraceSize * sizeof(Event); DCHECK_LT(p, kTraceMemBegin + kTraceMemSize); return p; } @@ -146,9 +152,6 @@ void internal_start_thread(void(*func)(void*), void *arg); // Says whether the addr relates to a global var. // Guesses with high probability, may yield both false positives and negatives. bool IsGlobalVar(uptr addr); -uptr GetTlsSize(); -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size); int ExtractResolvFDs(void *state, int *fds, int nfd); } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_platform_linux.cc b/libsanitizer/tsan/tsan_platform_linux.cc index f7b05f2bf8f..f13e3c893b3 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cc +++ b/libsanitizer/tsan/tsan_platform_linux.cc @@ -10,7 +10,9 @@ // Linux-specific code. //===----------------------------------------------------------------------===// -#ifdef __linux__ + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_LINUX #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" @@ -19,7 +21,6 @@ #include "tsan_rtl.h" #include "tsan_flags.h" -#include <asm/prctl.h> #include <fcntl.h> #include <pthread.h> #include <signal.h> @@ -40,11 +41,14 @@ #include <dlfcn.h> #define __need_res_state #include <resolv.h> +#include <malloc.h> -extern "C" int arch_prctl(int code, __sanitizer::uptr *addr); +extern "C" struct mallinfo __libc_mallinfo(); namespace __tsan { +const uptr kPageSize = 4096; + #ifndef TSAN_GO ScopedInRtl::ScopedInRtl() : thr_(cur_thread()) { @@ -66,8 +70,38 @@ ScopedInRtl::~ScopedInRtl() { } #endif -uptr GetShadowMemoryConsumption() { - return 0; +void FillProfileCallback(uptr start, uptr rss, bool file, + uptr *mem, uptr stats_size) { + CHECK_EQ(7, stats_size); + mem[6] += rss; // total + start >>= 40; + if (start < 0x10) // shadow + mem[0] += rss; + else if (start >= 0x20 && start < 0x30) // compat modules + mem[file ? 1 : 2] += rss; + else if (start >= 0x7e) // modules + mem[file ? 1 : 2] += rss; + else if (start >= 0x60 && start < 0x62) // traces + mem[3] += rss; + else if (start >= 0x7d && start < 0x7e) // heap + mem[4] += rss; + else // other + mem[5] += rss; +} + +void WriteMemoryProfile(char *buf, uptr buf_size) { + uptr mem[7] = {}; + __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); + char *buf_pos = buf; + char *buf_end = buf + buf_size; + buf_pos += internal_snprintf(buf_pos, buf_end - buf_pos, + "RSS %zd MB: shadow:%zd file:%zd mmap:%zd trace:%zd heap:%zd other:%zd\n", + mem[6] >> 20, mem[0] >> 20, mem[1] >> 20, mem[2] >> 20, + mem[3] >> 20, mem[4] >> 20, mem[5] >> 20); + struct mallinfo mi = __libc_mallinfo(); + buf_pos += internal_snprintf(buf_pos, buf_end - buf_pos, + "mallinfo: arena=%d mmap=%d fordblks=%d keepcost=%d\n", + mi.arena >> 20, mi.hblkhd >> 20, mi.fordblks >> 20, mi.keepcost >> 20); } void FlushShadowMemory() { @@ -89,6 +123,63 @@ static void ProtectRange(uptr beg, uptr end) { #endif #ifndef TSAN_GO +// Mark shadow for .rodata sections with the special kShadowRodata marker. +// Accesses to .rodata can't race, so this saves time, memory and trace space. +static void MapRodata() { + // First create temp file. + const char *tmpdir = GetEnv("TMPDIR"); + if (tmpdir == 0) + tmpdir = GetEnv("TEST_TMPDIR"); +#ifdef P_tmpdir + if (tmpdir == 0) + tmpdir = P_tmpdir; +#endif + if (tmpdir == 0) + return; + char filename[256]; + internal_snprintf(filename, sizeof(filename), "%s/tsan.rodata.%d", + tmpdir, (int)internal_getpid()); + uptr openrv = internal_open(filename, O_RDWR | O_CREAT | O_EXCL, 0600); + if (internal_iserror(openrv)) + return; + fd_t fd = openrv; + // Fill the file with kShadowRodata. + const uptr kMarkerSize = 512 * 1024 / sizeof(u64); + InternalScopedBuffer<u64> marker(kMarkerSize); + for (u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) + *p = kShadowRodata; + internal_write(fd, marker.data(), marker.size()); + // Map the file into memory. + uptr page = internal_mmap(0, kPageSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); + if (internal_iserror(page)) { + internal_close(fd); + internal_unlink(filename); + return; + } + // Map the file into shadow of .rodata sections. + MemoryMappingLayout proc_maps(/*cache_enabled*/true); + uptr start, end, offset, prot; + char name[128]; + while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) { + if (name[0] != 0 && name[0] != '[' + && (prot & MemoryMappingLayout::kProtectionRead) + && (prot & MemoryMappingLayout::kProtectionExecute) + && !(prot & MemoryMappingLayout::kProtectionWrite) + && IsAppMem(start)) { + // Assume it's .rodata + char *shadow_start = (char*)MemToShadow(start); + char *shadow_end = (char*)MemToShadow(end); + for (char *p = shadow_start; p < shadow_end; p += marker.size()) { + internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p), + PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); + } + } + } + internal_close(fd); + internal_unlink(filename); +} + void InitializeShadowMemory() { uptr shadow = (uptr)MmapFixedNoReserve(kLinuxShadowBeg, kLinuxShadowEnd - kLinuxShadowBeg); @@ -115,6 +206,8 @@ void InitializeShadowMemory() { kLinuxAppMemBeg, kLinuxAppMemEnd, (kLinuxAppMemEnd - kLinuxAppMemBeg) >> 30); DPrintf("stack %zx\n", (uptr)&shadow); + + MapRodata(); } #endif @@ -124,10 +217,11 @@ static uptr g_data_end; #ifndef TSAN_GO static void CheckPIE() { // Ensure that the binary is indeed compiled with -pie. - MemoryMappingLayout proc_maps; + MemoryMappingLayout proc_maps(true); uptr start, end; if (proc_maps.Next(&start, &end, - /*offset*/0, /*filename*/0, /*filename_size*/0)) { + /*offset*/0, /*filename*/0, /*filename_size*/0, + /*protection*/0)) { if ((u64)start < kLinuxAppMemBeg) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory (" "something is mapped at 0x%zx < 0x%zx)\n", @@ -140,11 +234,12 @@ static void CheckPIE() { } static void InitDataSeg() { - MemoryMappingLayout proc_maps; + MemoryMappingLayout proc_maps(true); uptr start, end, offset; char name[128]; bool prev_is_data = false; - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name))) { + while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), + /*protection*/ 0)) { DPrintf("%p-%p %p %s\n", start, end, offset, name); bool is_data = offset != 0 && name[0] != 0; // BSS may get merged with [heap] in /proc/self/maps. This is not very @@ -163,27 +258,6 @@ static void InitDataSeg() { CHECK_LT((uptr)&g_data_start, g_data_end); } -static uptr g_tls_size; - -#ifdef __i386__ -# define INTERNAL_FUNCTION __attribute__((regparm(3), stdcall)) -#else -# define INTERNAL_FUNCTION -#endif - -static int InitTlsSize() { - typedef void (*get_tls_func)(size_t*, size_t*) INTERNAL_FUNCTION; - get_tls_func get_tls; - void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info"); - CHECK_EQ(sizeof(get_tls), sizeof(get_tls_static_info_ptr)); - internal_memcpy(&get_tls, &get_tls_static_info_ptr, - sizeof(get_tls_static_info_ptr)); - CHECK_NE(get_tls, 0); - size_t tls_size = 0; - size_t tls_align = 0; - get_tls(&tls_size, &tls_align); - return tls_size; -} #endif // #ifndef TSAN_GO static rlim_t getlim(int res) { @@ -238,53 +312,12 @@ const char *InitializePlatform() { #ifndef TSAN_GO CheckPIE(); - g_tls_size = (uptr)InitTlsSize(); + InitTlsSize(); InitDataSeg(); #endif return GetEnv(kTsanOptionsEnv); } -void FinalizePlatform() { - fflush(0); -} - -uptr GetTlsSize() { -#ifndef TSAN_GO - return g_tls_size; -#else - return 0; -#endif -} - -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { -#ifndef TSAN_GO - arch_prctl(ARCH_GET_FS, tls_addr); - *tls_addr -= g_tls_size; - *tls_size = g_tls_size; - - uptr stack_top, stack_bottom; - GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); - *stk_addr = stack_bottom; - *stk_size = stack_top - stack_bottom; - - if (!main) { - // If stack and tls intersect, make them non-intersecting. - if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) { - CHECK_GT(*tls_addr + *tls_size, *stk_addr); - CHECK_LE(*tls_addr + *tls_size, *stk_addr + *stk_size); - *stk_size -= *tls_size; - *tls_addr = *stk_addr + *stk_size; - } - } -#else - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -#endif -} - bool IsGlobalVar(uptr addr) { return g_data_start && addr >= g_data_start && addr < g_data_end; } @@ -304,4 +337,4 @@ int ExtractResolvFDs(void *state, int *fds, int nfd) { } // namespace __tsan -#endif // #ifdef __linux__ +#endif // SANITIZER_LINUX diff --git a/libsanitizer/tsan/tsan_platform_mac.cc b/libsanitizer/tsan/tsan_platform_mac.cc index b247468c829..3dca611dc92 100644 --- a/libsanitizer/tsan/tsan_platform_mac.cc +++ b/libsanitizer/tsan/tsan_platform_mac.cc @@ -10,7 +10,8 @@ // Mac-specific code. //===----------------------------------------------------------------------===// -#ifdef __APPLE__ +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" @@ -87,18 +88,6 @@ void FinalizePlatform() { fflush(0); } -uptr GetTlsSize() { - return 0; -} - -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -} - } // namespace __tsan -#endif // #ifdef __APPLE__ +#endif // SANITIZER_MAC diff --git a/libsanitizer/tsan/tsan_platform_windows.cc b/libsanitizer/tsan/tsan_platform_windows.cc index 376dc08688b..6e49ef42f0c 100644 --- a/libsanitizer/tsan/tsan_platform_windows.cc +++ b/libsanitizer/tsan/tsan_platform_windows.cc @@ -10,7 +10,8 @@ // Windows-specific code. //===----------------------------------------------------------------------===// -#ifdef _WIN32 +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_WINDOWS #include "tsan_platform.h" @@ -39,18 +40,6 @@ void FinalizePlatform() { fflush(0); } -uptr GetTlsSize() { - return 0; -} - -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -} - } // namespace __tsan -#endif // #ifdef _WIN32 +#endif // SANITIZER_WINDOWS diff --git a/libsanitizer/tsan/tsan_report.cc b/libsanitizer/tsan/tsan_report.cc index 098d8262ba1..15ab22b4ba7 100644 --- a/libsanitizer/tsan/tsan_report.cc +++ b/libsanitizer/tsan/tsan_report.cc @@ -11,16 +11,35 @@ #include "tsan_report.h" #include "tsan_platform.h" #include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_report_decorator.h" namespace __tsan { +class Decorator: private __sanitizer::AnsiColorDecorator { + public: + Decorator() : __sanitizer::AnsiColorDecorator(PrintsToTtyCached()) { } + const char *Warning() { return Red(); } + const char *EndWarning() { return Default(); } + const char *Access() { return Blue(); } + const char *EndAccess() { return Default(); } + const char *ThreadDescription() { return Cyan(); } + const char *EndThreadDescription() { return Default(); } + const char *Location() { return Green(); } + const char *EndLocation() { return Default(); } + const char *Sleep() { return Yellow(); } + const char *EndSleep() { return Default(); } + const char *Mutex() { return Magenta(); } + const char *EndMutex() { return Default(); } +}; + ReportDesc::ReportDesc() : stacks(MBlockReportStack) , mops(MBlockReportMop) , locs(MBlockReportLoc) , mutexes(MBlockReportMutex) , threads(MBlockReportThread) - , sleep() { + , sleep() + , count() { } ReportMop::ReportMop() @@ -44,6 +63,8 @@ const char *thread_name(char *buf, int tid) { static const char *ReportTypeString(ReportType typ) { if (typ == ReportTypeRace) return "data race"; + if (typ == ReportTypeVptrRace) + return "data race on vptr (ctor/dtor vs virtual call)"; if (typ == ReportTypeUseAfterFree) return "heap-use-after-free"; if (typ == ReportTypeThreadLeak) @@ -92,18 +113,24 @@ static const char *MopDesc(bool first, bool write, bool atomic) { } static void PrintMop(const ReportMop *mop, bool first) { + Decorator d; char thrbuf[kThreadBufSize]; + Printf("%s", d.Access()); Printf(" %s of size %d at %p by %s", MopDesc(first, mop->write, mop->atomic), mop->size, (void*)mop->addr, thread_name(thrbuf, mop->tid)); PrintMutexSet(mop->mset); Printf(":\n"); + Printf("%s", d.EndAccess()); PrintStack(mop->stack); } static void PrintLocation(const ReportLocation *loc) { + Decorator d; char thrbuf[kThreadBufSize]; + bool print_stack = false; + Printf("%s", d.Location()); if (loc->type == ReportLocationGlobal) { Printf(" Location is global '%s' of size %zu at %zx (%s+%p)\n\n", loc->name, loc->size, loc->addr, loc->module, loc->offset); @@ -111,7 +138,7 @@ static void PrintLocation(const ReportLocation *loc) { char thrbuf[kThreadBufSize]; Printf(" Location is heap block of size %zu at %p allocated by %s:\n", loc->size, loc->addr, thread_name(thrbuf, loc->tid)); - PrintStack(loc->stack); + print_stack = true; } else if (loc->type == ReportLocationStack) { Printf(" Location is stack of %s.\n\n", thread_name(thrbuf, loc->tid)); } else if (loc->type == ReportLocationTLS) { @@ -119,24 +146,34 @@ static void PrintLocation(const ReportLocation *loc) { } else if (loc->type == ReportLocationFD) { Printf(" Location is file descriptor %d created by %s at:\n", loc->fd, thread_name(thrbuf, loc->tid)); - PrintStack(loc->stack); + print_stack = true; } + Printf("%s", d.EndLocation()); + if (print_stack) + PrintStack(loc->stack); } static void PrintMutex(const ReportMutex *rm) { + Decorator d; if (rm->destroyed) { + Printf("%s", d.Mutex()); Printf(" Mutex M%llu is already destroyed.\n\n", rm->id); + Printf("%s", d.EndMutex()); } else { + Printf("%s", d.Mutex()); Printf(" Mutex M%llu created at:\n", rm->id); + Printf("%s", d.EndMutex()); PrintStack(rm->stack); } } static void PrintThread(const ReportThread *rt) { + Decorator d; if (rt->id == 0) // Little sense in describing the main thread. return; + Printf("%s", d.ThreadDescription()); Printf(" Thread T%d", rt->id); - if (rt->name) + if (rt->name && rt->name[0] != '\0') Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; Printf(" (tid=%zu, %s) created by %s", @@ -145,11 +182,15 @@ static void PrintThread(const ReportThread *rt) { if (rt->stack) Printf(" at:"); Printf("\n"); + Printf("%s", d.EndThreadDescription()); PrintStack(rt->stack); } static void PrintSleep(const ReportStack *s) { + Decorator d; + Printf("%s", d.Sleep()); Printf(" As if synchronized via sleep:\n"); + Printf("%s", d.EndSleep()); PrintStack(s); } @@ -172,9 +213,13 @@ ReportStack *SkipTsanInternalFrames(ReportStack *ent) { } void PrintReport(const ReportDesc *rep) { + Decorator d; Printf("==================\n"); const char *rep_typ_str = ReportTypeString(rep->typ); - Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str, GetPid()); + Printf("%s", d.Warning()); + Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str, + (int)internal_getpid()); + Printf("%s", d.EndWarning()); for (uptr i = 0; i < rep->stacks.Size(); i++) { if (i) @@ -197,37 +242,46 @@ void PrintReport(const ReportDesc *rep) { for (uptr i = 0; i < rep->threads.Size(); i++) PrintThread(rep->threads[i]); + if (rep->typ == ReportTypeThreadLeak && rep->count > 1) + Printf(" And %d more similar thread leaks.\n\n", rep->count - 1); + if (ReportStack *ent = SkipTsanInternalFrames(ChooseSummaryStack(rep))) ReportErrorSummary(rep_typ_str, ent->file, ent->line, ent->func); Printf("==================\n"); } -#else +#else // #ifndef TSAN_GO + +const int kMainThreadId = 1; void PrintStack(const ReportStack *ent) { if (ent == 0) { - Printf(" [failed to restore the stack]\n\n"); + Printf(" [failed to restore the stack]\n"); return; } for (int i = 0; ent; ent = ent->next, i++) { Printf(" %s()\n %s:%d +0x%zx\n", ent->func, ent->file, ent->line, (void*)ent->offset); } - Printf("\n"); } static void PrintMop(const ReportMop *mop, bool first) { - Printf("%s by goroutine %d:\n", + Printf("\n"); + Printf("%s by ", (first ? (mop->write ? "Write" : "Read") - : (mop->write ? "Previous write" : "Previous read")), - mop->tid); + : (mop->write ? "Previous write" : "Previous read"))); + if (mop->tid == kMainThreadId) + Printf("main goroutine:\n"); + else + Printf("goroutine %d:\n", mop->tid); PrintStack(mop->stack); } static void PrintThread(const ReportThread *rt) { - if (rt->id == 0) // Little sense in describing the main thread. + if (rt->id == kMainThreadId) return; + Printf("\n"); Printf("Goroutine %d (%s) created at:\n", rt->id, rt->running ? "running" : "finished"); PrintStack(rt->stack); @@ -235,7 +289,7 @@ static void PrintThread(const ReportThread *rt) { void PrintReport(const ReportDesc *rep) { Printf("==================\n"); - Printf("WARNING: DATA RACE\n"); + Printf("WARNING: DATA RACE"); for (uptr i = 0; i < rep->mops.Size(); i++) PrintMop(rep->mops[i], i == 0); for (uptr i = 0; i < rep->threads.Size(); i++) diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h index eae2b3c721f..c0eef9eb023 100644 --- a/libsanitizer/tsan/tsan_report.h +++ b/libsanitizer/tsan/tsan_report.h @@ -18,6 +18,7 @@ namespace __tsan { enum ReportType { ReportTypeRace, + ReportTypeVptrRace, ReportTypeUseAfterFree, ReportTypeThreadLeak, ReportTypeMutexDestroyLocked, @@ -99,6 +100,7 @@ class ReportDesc { Vector<ReportMutex*> mutexes; Vector<ReportThread*> threads; ReportStack *sleep; + int count; ReportDesc(); ~ReportDesc(); diff --git a/libsanitizer/tsan/tsan_rtl.cc b/libsanitizer/tsan/tsan_rtl.cc index 673a355f1dc..7f18064e957 100644 --- a/libsanitizer/tsan/tsan_rtl.cc +++ b/libsanitizer/tsan/tsan_rtl.cc @@ -21,6 +21,7 @@ #include "tsan_rtl.h" #include "tsan_mman.h" #include "tsan_suppressions.h" +#include "tsan_symbolize.h" volatile int __tsan_resumed = 0; @@ -45,15 +46,33 @@ Context *CTX() { return ctx; } +static char thread_registry_placeholder[sizeof(ThreadRegistry)]; + +static ThreadContextBase *CreateThreadContext(u32 tid) { + // Map thread trace when context is created. + MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event)); + MapThreadTrace(GetThreadTraceHeader(tid), sizeof(Trace)); + new(ThreadTrace(tid)) Trace(); + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); + return new(mem) ThreadContext(tid); +} + +#ifndef TSAN_GO +static const u32 kThreadQuarantineSize = 16; +#else +static const u32 kThreadQuarantineSize = 64; +#endif + Context::Context() : initialized() , report_mtx(MutexTypeReport, StatMtxReport) , nreported() , nmissed_expected() - , thread_mtx(MutexTypeThreads, StatMtxThreads) + , thread_registry(new(thread_registry_placeholder) ThreadRegistry( + CreateThreadContext, kMaxTid, kThreadQuarantineSize)) , racy_stacks(MBlockRacyStacks) , racy_addresses(MBlockRacyAddresses) - , fired_suppressions(MBlockRacyAddresses) { + , fired_suppressions(8) { } // The objects are allocated in TLS, so one may rely on zero-initialization. @@ -63,10 +82,12 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, : fast_state(tid, epoch) // Do not touch these, rely on zero initialization, // they may be accessed before the ctor. - // , fast_ignore_reads() - // , fast_ignore_writes() + // , ignore_reads_and_writes() // , in_rtl() , shadow_stack_pos(&shadow_stack[0]) +#ifndef TSAN_GO + , jmp_bufs(MBlockJmpBuf) +#endif , tid(tid) , unique_id(unique_id) , stk_addr(stk_addr) @@ -75,94 +96,74 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, , tls_size(tls_size) { } -ThreadContext::ThreadContext(int tid) - : tid(tid) - , unique_id() - , os_id() - , user_id() - , thr() - , status(ThreadStatusInvalid) - , detached() - , reuse_count() - , epoch0() - , epoch1() - , dead_info() - , dead_next() - , name() { -} - -static void WriteMemoryProfile(char *buf, uptr buf_size, int num) { - uptr shadow = GetShadowMemoryConsumption(); - - int nthread = 0; - int nlivethread = 0; - uptr threadmem = 0; - { - Lock l(&ctx->thread_mtx); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0) - continue; - nthread += 1; - threadmem += sizeof(ThreadContext); - if (tctx->status != ThreadStatusRunning) - continue; - nlivethread += 1; - threadmem += sizeof(ThreadState); - } - } - - uptr nsync = 0; - uptr syncmem = CTX()->synctab.GetMemoryConsumption(&nsync); - - internal_snprintf(buf, buf_size, "%d: shadow=%zuMB" - " thread=%zuMB(total=%d/live=%d)" - " sync=%zuMB(cnt=%zu)\n", - num, - shadow >> 20, - threadmem >> 20, nthread, nlivethread, - syncmem >> 20, nsync); +static void MemoryProfiler(Context *ctx, fd_t fd, int i) { + uptr n_threads; + uptr n_running_threads; + ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads); + InternalScopedBuffer<char> buf(4096); + internal_snprintf(buf.data(), buf.size(), "%d: nthr=%d nlive=%d\n", + i, n_threads, n_running_threads); + internal_write(fd, buf.data(), internal_strlen(buf.data())); + WriteMemoryProfile(buf.data(), buf.size()); + internal_write(fd, buf.data(), internal_strlen(buf.data())); } -static void MemoryProfileThread(void *arg) { +static void BackgroundThread(void *arg) { ScopedInRtl in_rtl; - fd_t fd = (fd_t)(uptr)arg; + Context *ctx = CTX(); + const u64 kMs2Ns = 1000 * 1000; + + fd_t mprof_fd = kInvalidFd; + if (flags()->profile_memory && flags()->profile_memory[0]) { + InternalScopedBuffer<char> filename(4096); + internal_snprintf(filename.data(), filename.size(), "%s.%d", + flags()->profile_memory, (int)internal_getpid()); + uptr openrv = OpenFile(filename.data(), true); + if (internal_iserror(openrv)) { + Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", + &filename[0]); + } else { + mprof_fd = openrv; + } + } + + u64 last_flush = NanoTime(); for (int i = 0; ; i++) { - InternalScopedBuffer<char> buf(4096); - WriteMemoryProfile(buf.data(), buf.size(), i); - internal_write(fd, buf.data(), internal_strlen(buf.data())); SleepForSeconds(1); - } -} + u64 now = NanoTime(); + + // Flush memory if requested. + if (flags()->flush_memory_ms) { + if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) { + FlushShadowMemory(); + last_flush = NanoTime(); + } + } -static void InitializeMemoryProfile() { - if (flags()->profile_memory == 0 || flags()->profile_memory[0] == 0) - return; - InternalScopedBuffer<char> filename(4096); - internal_snprintf(filename.data(), filename.size(), "%s.%d", - flags()->profile_memory, GetPid()); - fd_t fd = OpenFile(filename.data(), true); - if (fd == kInvalidFd) { - Printf("Failed to open memory profile file '%s'\n", &filename[0]); - Die(); - } - internal_start_thread(&MemoryProfileThread, (void*)(uptr)fd); -} + // Write memory profile if requested. + if (mprof_fd != kInvalidFd) + MemoryProfiler(ctx, mprof_fd, i); -static void MemoryFlushThread(void *arg) { - ScopedInRtl in_rtl; - for (int i = 0; ; i++) { - SleepForMillis(flags()->flush_memory_ms); - FlushShadowMemory(); +#ifndef TSAN_GO + // Flush symbolizer cache if requested. + if (flags()->flush_symbolizer_ms > 0) { + u64 last = atomic_load(&ctx->last_symbolize_time_ns, + memory_order_relaxed); + if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { + Lock l(&ctx->report_mtx); + SpinMutexLock l2(&CommonSanitizerReportMutex); + SymbolizeFlush(); + atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); + } + } +#endif } } -static void InitializeMemoryFlush() { - if (flags()->flush_memory_ms == 0) - return; - if (flags()->flush_memory_ms < 100) - flags()->flush_memory_ms = 100; - internal_start_thread(&MemoryFlushThread, 0); +void DontNeedShadowFor(uptr addr, uptr size) { + uptr shadow_beg = MemToShadow(addr); + uptr shadow_end = MemToShadow(addr + size); + FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); } void MapShadow(uptr addr, uptr size) { @@ -201,9 +202,6 @@ void Initialize(ThreadState *thr) { #ifndef TSAN_GO InitializeShadowMemory(); #endif - ctx->dead_list_size = 0; - ctx->dead_list_head = 0; - ctx->dead_list_tail = 0; InitializeFlags(&ctx->flags, env); // Setup correct file descriptor for error reports. if (internal_strcmp(flags()->log_path, "stdout") == 0) @@ -217,32 +215,30 @@ void Initialize(ThreadState *thr) { // Initialize external symbolizer before internal threads are started. const char *external_symbolizer = flags()->external_symbolizer_path; if (external_symbolizer != 0 && external_symbolizer[0] != '\0') { - if (!InitializeExternalSymbolizer(external_symbolizer)) { + if (!getSymbolizer()->InitializeExternal(external_symbolizer)) { Printf("Failed to start external symbolizer: '%s'\n", external_symbolizer); Die(); } } #endif - InitializeMemoryProfile(); - InitializeMemoryFlush(); + internal_start_thread(&BackgroundThread, 0); if (ctx->flags.verbosity) Printf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", - GetPid()); + (int)internal_getpid()); // Initialize thread 0. - ctx->thread_seq = 0; int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetPid()); + ThreadStart(thr, tid, internal_getpid()); CHECK_EQ(thr->in_rtl, 1); ctx->initialized = true; if (flags()->stop_on_start) { Printf("ThreadSanitizer is suspended at startup (pid %d)." " Call __tsan_resume().\n", - GetPid()); + (int)internal_getpid()); while (__tsan_resumed == 0) {} } } @@ -257,6 +253,8 @@ int Finalize(ThreadState *thr) { // Wait for pending reports. ctx->report_mtx.Lock(); + CommonSanitizerReportMutex.Lock(); + CommonSanitizerReportMutex.Unlock(); ctx->report_mtx.Unlock(); #ifndef TSAN_GO @@ -281,6 +279,13 @@ int Finalize(ThreadState *thr) { ctx->nmissed_expected); } + if (flags()->print_suppressions) + PrintMatchedSuppressions(); +#ifndef TSAN_GO + if (flags()->print_benign) + PrintMatchedBenignRaces(); +#endif + failed = OnFinalize(failed); StatAggregate(ctx->stat, thr->stat); @@ -307,15 +312,20 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) { void TraceSwitch(ThreadState *thr) { thr->nomalloc++; ScopedInRtl in_rtl; - Lock l(&thr->trace.mtx); + Trace *thr_trace = ThreadTrace(thr->tid); + Lock l(&thr_trace->mtx); unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); - TraceHeader *hdr = &thr->trace.headers[trace]; + TraceHeader *hdr = &thr_trace->headers[trace]; hdr->epoch0 = thr->fast_state.epoch(); hdr->stack0.ObtainCurrent(thr, 0); hdr->mset0 = thr->mset; thr->nomalloc--; } +Trace *ThreadTrace(int tid) { + return (Trace*)GetThreadTraceHeader(tid); +} + uptr TraceTopPC(ThreadState *thr) { Event *events = (Event*)GetThreadTrace(thr->tid); uptr pc = events[thr->fast_state.GetTracePos()]; @@ -341,18 +351,18 @@ extern "C" void __tsan_report_race() { #endif ALWAYS_INLINE -static Shadow LoadShadow(u64 *p) { +Shadow LoadShadow(u64 *p) { u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); return Shadow(raw); } ALWAYS_INLINE -static void StoreShadow(u64 *sp, u64 s) { +void StoreShadow(u64 *sp, u64 s) { atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); } ALWAYS_INLINE -static void StoreIfNotYetStored(u64 *sp, u64 *s) { +void StoreIfNotYetStored(u64 *sp, u64 *s) { StoreShadow(sp, *s); *s = 0; } @@ -377,7 +387,7 @@ static inline bool HappensBefore(Shadow old, ThreadState *thr) { return thr->clock.get(old.TidWithIgnore()) >= old.epoch(); } -ALWAYS_INLINE +ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr, int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem, Shadow cur) { @@ -451,7 +461,28 @@ void MemoryAccessImpl(ThreadState *thr, uptr addr, return; } -ALWAYS_INLINE +void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, + int size, bool kAccessIsWrite, bool kIsAtomic) { + while (size) { + int size1 = 1; + int kAccessSizeLog = kSizeLog1; + if (size >= 8 && (addr & ~7) == ((addr + 8) & ~7)) { + size1 = 8; + kAccessSizeLog = kSizeLog8; + } else if (size >= 4 && (addr & ~7) == ((addr + 4) & ~7)) { + size1 = 4; + kAccessSizeLog = kSizeLog4; + } else if (size >= 2 && (addr & ~7) == ((addr + 2) & ~7)) { + size1 = 2; + kAccessSizeLog = kSizeLog2; + } + MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic); + addr += size1; + size -= size1; + } +} + +ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) { u64 *shadow_mem = (u64*)MemToShadow(addr); @@ -472,6 +503,16 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif + if (*shadow_mem == kShadowRodata) { + // Access to .rodata section, no races here. + // Measurements show that it can be 10-20% of all memory accesses. + StatInc(thr, StatMop); + StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); + StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); + StatInc(thr, StatMopRodata); + return; + } + FastState fast_state = thr->fast_state; if (fast_state.GetIgnoreBit()) return; @@ -492,6 +533,8 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, u64 val) { + (void)thr; + (void)pc; if (size == 0) return; // FIXME: fix me. @@ -508,23 +551,44 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, // let it just crash as usual. if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) return; - (void)thr; - (void)pc; - // Some programs mmap like hundreds of GBs but actually used a small part. - // So, it's better to report a false positive on the memory - // then to hang here senselessly. - const uptr kMaxResetSize = 4ull*1024*1024*1024; - if (size > kMaxResetSize) - size = kMaxResetSize; + // Don't want to touch lots of shadow memory. + // If a program maps 10MB stack, there is no need reset the whole range. size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); - u64 *p = (u64*)MemToShadow(addr); - CHECK(IsShadowMem((uptr)p)); - CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); - // FIXME: may overwrite a part outside the region - for (uptr i = 0; i < size * kShadowCnt / kShadowCell;) { - p[i++] = val; - for (uptr j = 1; j < kShadowCnt; j++) - p[i++] = 0; + // UnmapOrDie/MmapFixedNoReserve does not work on Windows, + // so we do it only for C/C++. + if (kGoMode || size < 64*1024) { + u64 *p = (u64*)MemToShadow(addr); + CHECK(IsShadowMem((uptr)p)); + CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); + // FIXME: may overwrite a part outside the region + for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { + p[i++] = val; + for (uptr j = 1; j < kShadowCnt; j++) + p[i++] = 0; + } + } else { + // The region is big, reset only beginning and end. + const uptr kPageSize = 4096; + u64 *begin = (u64*)MemToShadow(addr); + u64 *end = begin + size / kShadowCell * kShadowCnt; + u64 *p = begin; + // Set at least first kPageSize/2 to page boundary. + while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { + *p++ = val; + for (uptr j = 1; j < kShadowCnt; j++) + *p++ = 0; + } + // Reset middle part. + u64 *p1 = p; + p = RoundDown(end, kPageSize); + UnmapOrDie((void*)p1, (uptr)p - (uptr)p1); + MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1); + // Set the ending. + while (p < end) { + *p++ = val; + for (uptr j = 1; j < kShadowCnt; j++) + *p++ = 0; + } } } @@ -533,10 +597,17 @@ void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { } void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { + // Processing more than 1k (4k of shadow) is expensive, + // can cause excessive memory consumption (user does not necessary touch + // the whole range) and most likely unnecessary. + if (size > 1024) + size = 1024; CHECK_EQ(thr->is_freeing, false); thr->is_freeing = true; MemoryAccessRange(thr, pc, addr, size, true); thr->is_freeing = false; + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); Shadow s(thr->fast_state); s.ClearIgnoreBit(); s.MarkAsFreed(); @@ -546,6 +617,8 @@ void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { } void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); Shadow s(thr->fast_state); s.ClearIgnoreBit(); s.SetWrite(true); @@ -553,7 +626,7 @@ void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { MemoryRangeSet(thr, pc, addr, size, s.raw()); } -ALWAYS_INLINE +ALWAYS_INLINE USED void FuncEntry(ThreadState *thr, uptr pc) { DCHECK_EQ(thr->in_rtl, 0); StatInc(thr, StatFuncEnter); @@ -583,7 +656,7 @@ void FuncEntry(ThreadState *thr, uptr pc) { thr->shadow_stack_pos++; } -ALWAYS_INLINE +ALWAYS_INLINE USED void FuncExit(ThreadState *thr) { DCHECK_EQ(thr->in_rtl, 0); StatInc(thr, StatFuncExit); @@ -598,13 +671,18 @@ void FuncExit(ThreadState *thr) { thr->shadow_stack_pos--; } -void IgnoreCtl(ThreadState *thr, bool write, bool begin) { - DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); - thr->ignore_reads_and_writes += begin ? 1 : -1; +void ThreadIgnoreBegin(ThreadState *thr) { + DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid); + thr->ignore_reads_and_writes++; CHECK_GE(thr->ignore_reads_and_writes, 0); - if (thr->ignore_reads_and_writes) - thr->fast_state.SetIgnoreBit(); - else + thr->fast_state.SetIgnoreBit(); +} + +void ThreadIgnoreEnd(ThreadState *thr) { + DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid); + thr->ignore_reads_and_writes--; + CHECK_GE(thr->ignore_reads_and_writes, 0); + if (thr->ignore_reads_and_writes == 0) thr->fast_state.ClearIgnoreBit(); } diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index e939921049a..2548f67b25c 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -24,8 +24,11 @@ #ifndef TSAN_RTL_H #define TSAN_RTL_H -#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_suppressions.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_flags.h" @@ -44,15 +47,73 @@ namespace __tsan { // Descriptor of user's memory block. struct MBlock { - Mutex mtx; - uptr size; - u32 alloc_tid; - u32 alloc_stack_id; - SyncVar *head; + /* + u64 mtx : 1; // must be first + u64 lst : 44; + u64 stk : 31; // on word boundary + u64 tid : kTidBits; + u64 siz : 128 - 1 - 31 - 44 - kTidBits; // 39 + */ + u64 raw[2]; + + void Init(uptr siz, u32 tid, u32 stk) { + raw[0] = raw[1] = 0; + raw[1] |= (u64)siz << ((1 + 44 + 31 + kTidBits) % 64); + raw[1] |= (u64)tid << ((1 + 44 + 31) % 64); + raw[0] |= (u64)stk << (1 + 44); + raw[1] |= (u64)stk >> (64 - 44 - 1); + DCHECK_EQ(Size(), siz); + DCHECK_EQ(Tid(), tid); + DCHECK_EQ(StackId(), stk); + } + + u32 Tid() const { + return GetLsb(raw[1] >> ((1 + 44 + 31) % 64), kTidBits); + } + + uptr Size() const { + return raw[1] >> ((1 + 31 + 44 + kTidBits) % 64); + } + + u32 StackId() const { + return (raw[0] >> (1 + 44)) | GetLsb(raw[1] << (64 - 44 - 1), 31); + } - MBlock() - : mtx(MutexTypeMBlock, StatMtxMBlock) { + SyncVar *ListHead() const { + return (SyncVar*)(GetLsb(raw[0] >> 1, 44) << 3); } + + void ListPush(SyncVar *v) { + SyncVar *lst = ListHead(); + v->next = lst; + u64 x = (u64)v ^ (u64)lst; + x = (x >> 3) << 1; + raw[0] ^= x; + DCHECK_EQ(ListHead(), v); + } + + SyncVar *ListPop() { + SyncVar *lst = ListHead(); + SyncVar *nxt = lst->next; + lst->next = 0; + u64 x = (u64)lst ^ (u64)nxt; + x = (x >> 3) << 1; + raw[0] ^= x; + DCHECK_EQ(ListHead(), nxt); + return lst; + } + + void ListReset() { + SyncVar *lst = ListHead(); + u64 x = (u64)lst; + x = (x >> 3) << 1; + raw[0] ^= x; + DCHECK_EQ(ListHead(), 0); + } + + void Lock(); + void Unlock(); + typedef GenericScopedLock<MBlock> ScopedLock; }; #ifndef TSAN_GO @@ -63,22 +124,11 @@ const uptr kAllocatorSpace = 0x7d0000000000ULL; #endif const uptr kAllocatorSize = 0x10000000000ULL; // 1T. -struct TsanMapUnmapCallback { - void OnMap(uptr p, uptr size) const { } - void OnUnmap(uptr p, uptr size) const { - // We are about to unmap a chunk of user memory. - // Mark the corresponding shadow memory as not needed. - uptr shadow_beg = MemToShadow(p); - uptr shadow_end = MemToShadow(p + size); - CHECK(IsAligned(shadow_end|shadow_beg, GetPageSizeCached())); - FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); - } -}; - +struct MapUnmapCallback; typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, sizeof(MBlock), - DefaultSizeClassMap> PrimaryAllocator; + DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<TsanMapUnmapCallback> SecondaryAllocator; +typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, SecondaryAllocator> Allocator; Allocator *allocator(); @@ -87,6 +137,8 @@ Allocator *allocator(); void TsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); +const u64 kShadowRodata = (u64)-1; // .rodata shadow marker + // FastState (from most significant bit): // ignore : 1 // tid : kTidBits @@ -332,6 +384,12 @@ class Shadow : public FastState { struct SignalContext; +struct JmpBuf { + uptr sp; + uptr mangled_sp; + uptr *shadow_stack_pos; +}; + // This struct is stored in TLS. struct ThreadState { FastState fast_state; @@ -354,7 +412,6 @@ struct ThreadState { uptr *shadow_stack_pos; u64 *racy_shadow_addr; u64 racy_state[2]; - Trace trace; #ifndef TSAN_GO // C/C++ uses embed shadow stack of fixed size. uptr shadow_stack[kShadowStackSize]; @@ -367,6 +424,8 @@ struct ThreadState { ThreadClock clock; #ifndef TSAN_GO AllocatorCache alloc_cache; + InternalAllocatorCache internal_alloc_cache; + Vector<JmpBuf> jmp_bufs; #endif u64 stat[StatCnt]; const int tid; @@ -375,6 +434,7 @@ struct ThreadState { bool in_symbolizer; bool is_alive; bool is_freeing; + bool is_vptr_access; const uptr stk_addr; const uptr stk_size; const uptr tls_addr; @@ -408,41 +468,30 @@ INLINE ThreadState *cur_thread() { } #endif -enum ThreadStatus { - ThreadStatusInvalid, // Non-existent thread, data is invalid. - ThreadStatusCreated, // Created but not yet running. - ThreadStatusRunning, // The thread is currently running. - ThreadStatusFinished, // Joinable thread is finished but not yet joined. - ThreadStatusDead // Joined, but some info (trace) is still alive. -}; - -// An info about a thread that is hold for some time after its termination. -struct ThreadDeadInfo { - Trace trace; -}; - -struct ThreadContext { - const int tid; - int unique_id; // Non-rolling thread id. - uptr os_id; // pid - uptr user_id; // Some opaque user thread id (e.g. pthread_t). +class ThreadContext : public ThreadContextBase { + public: + explicit ThreadContext(int tid); + ~ThreadContext(); ThreadState *thr; - ThreadStatus status; - bool detached; - int reuse_count; +#ifdef TSAN_GO + StackTrace creation_stack; +#else + u32 creation_stack_id; +#endif SyncClock sync; // Epoch at which the thread had started. // If we see an event from the thread stamped by an older epoch, // the event is from a dead thread that shared tid with this thread. u64 epoch0; u64 epoch1; - StackTrace creation_stack; - int creation_tid; - ThreadDeadInfo *dead_info; - ThreadContext *dead_next; // In dead thread list. - char *name; // As annotated by user. - explicit ThreadContext(int tid); + // Override superclass callbacks. + void OnDead(); + void OnJoined(void *arg); + void OnFinished(); + void OnStarted(void *arg); + void OnCreated(void *arg); + void OnReset(); }; struct RacyStacks { @@ -464,6 +513,7 @@ struct RacyAddress { struct FiredSuppression { ReportType type; uptr pc; + Suppression *supp; }; struct Context { @@ -476,20 +526,14 @@ struct Context { Mutex report_mtx; int nreported; int nmissed_expected; + atomic_uint64_t last_symbolize_time_ns; - Mutex thread_mtx; - unsigned thread_seq; - unsigned unique_thread_seq; - int alive_threads; - int max_alive_threads; - ThreadContext *threads[kMaxTid]; - int dead_list_size; - ThreadContext* dead_list_head; - ThreadContext* dead_list_tail; + ThreadRegistry *thread_registry; Vector<RacyStacks> racy_stacks; Vector<RacyAddress> racy_addresses; - Vector<FiredSuppression> fired_suppressions; + // Number of fired suppressions may be large enough. + InternalMmapVector<FiredSuppression> fired_suppressions; Flags flags; @@ -520,6 +564,7 @@ class ScopedReport { void AddMutex(const SyncVar *s); void AddLocation(uptr addr, uptr size); void AddSleep(u32 stack_id); + void SetCount(int count); const ReportDesc *GetReport() const; @@ -537,13 +582,18 @@ void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset); void StatAggregate(u64 *dst, u64 *src); void StatOutput(u64 *stat); -void ALWAYS_INLINE INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { +void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { if (kCollectStats) thr->stat[typ] += n; } +void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { + if (kCollectStats) + thr->stat[typ] = n; +} void MapShadow(uptr addr, uptr size); void MapThreadTrace(uptr addr, uptr size); +void DontNeedShadowFor(uptr addr, uptr size); void InitializeShadowMemory(); void InitializeInterceptors(); void InitializeDynamicAnnotations(); @@ -552,11 +602,13 @@ void ReportRace(ThreadState *thr); bool OutputReport(Context *ctx, const ScopedReport &srep, const ReportStack *suppress_stack1 = 0, - const ReportStack *suppress_stack2 = 0); + const ReportStack *suppress_stack2 = 0, + const ReportLocation *suppress_loc = 0); bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, const StackTrace &trace); bool IsExpectedReport(uptr addr, uptr size); +void PrintMatchedBenignRaces(); bool FrameIsInternal(const ReportStack *frame); ReportStack *SkipTsanInternalFrames(ReportStack *ent); @@ -592,28 +644,30 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, bool is_write); void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr, uptr size, uptr step, bool is_write); +void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, + int size, bool kAccessIsWrite, bool kIsAtomic); const int kSizeLog1 = 0; const int kSizeLog2 = 1; const int kSizeLog4 = 2; const int kSizeLog8 = 3; -void ALWAYS_INLINE INLINE MemoryRead(ThreadState *thr, uptr pc, +void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog) { MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false); } -void ALWAYS_INLINE INLINE MemoryWrite(ThreadState *thr, uptr pc, +void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog) { MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false); } -void ALWAYS_INLINE INLINE MemoryReadAtomic(ThreadState *thr, uptr pc, +void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog) { MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true); } -void ALWAYS_INLINE INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, +void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog) { MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true); } @@ -621,7 +675,8 @@ void ALWAYS_INLINE INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); -void IgnoreCtl(ThreadState *thr, bool write, bool begin); +void ThreadIgnoreBegin(ThreadState *thr); +void ThreadIgnoreEnd(ThreadState *thr); void FuncEntry(ThreadState *thr, uptr pc); void FuncExit(ThreadState *thr); @@ -640,8 +695,8 @@ void ProcessPendingSignals(ThreadState *thr); void MutexCreate(ThreadState *thr, uptr pc, uptr addr, bool rw, bool recursive, bool linker_init); void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); -void MutexLock(ThreadState *thr, uptr pc, uptr addr); -void MutexUnlock(ThreadState *thr, uptr pc, uptr addr); +void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1); +int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false); void MutexReadLock(ThreadState *thr, uptr pc, uptr addr); void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); @@ -677,9 +732,10 @@ void TraceSwitch(ThreadState *thr); uptr TraceTopPC(ThreadState *thr); uptr TraceSize(); uptr TraceParts(); +Trace *ThreadTrace(int tid); extern "C" void __tsan_trace_switch(); -void ALWAYS_INLINE INLINE TraceAddEvent(ThreadState *thr, FastState fs, +void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, EventType typ, u64 addr) { DCHECK_GE((int)typ, 0); DCHECK_LE((int)typ, 7); diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cc b/libsanitizer/tsan/tsan_rtl_mutex.cc index 22a71503c5c..d274a7a8cc5 100644 --- a/libsanitizer/tsan/tsan_rtl_mutex.cc +++ b/libsanitizer/tsan/tsan_rtl_mutex.cc @@ -61,7 +61,7 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { && s->owner_tid != SyncVar::kInvalidTid && !s->is_broken) { s->is_broken = true; - Lock l(&ctx->thread_mtx); + ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeMutexDestroyLocked); rep.AddMutex(s); StackTrace trace; @@ -77,9 +77,10 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { DestroyAndFree(s); } -void MutexLock(ThreadState *thr, uptr pc, uptr addr) { +void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec) { CHECK_GT(thr->in_rtl, 0); - DPrintf("#%d: MutexLock %zx\n", thr->tid, addr); + DPrintf("#%d: MutexLock %zx rec=%d\n", thr->tid, addr, rec); + CHECK_GT(rec, 0); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); @@ -92,7 +93,7 @@ void MutexLock(ThreadState *thr, uptr pc, uptr addr) { } else if (s->owner_tid == thr->tid) { CHECK_GT(s->recursion, 0); } else { - Printf("ThreadSanitizer WARNING: double lock\n"); + Printf("ThreadSanitizer WARNING: double lock of mutex %p\n", addr); PrintCurrentStack(thr, pc); } if (s->recursion == 0) { @@ -105,33 +106,36 @@ void MutexLock(ThreadState *thr, uptr pc, uptr addr) { } else if (!s->is_recursive) { StatInc(thr, StatMutexRecLock); } - s->recursion++; + s->recursion += rec; thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); s->mtx.Unlock(); } -void MutexUnlock(ThreadState *thr, uptr pc, uptr addr) { +int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) { CHECK_GT(thr->in_rtl, 0); - DPrintf("#%d: MutexUnlock %zx\n", thr->tid, addr); + DPrintf("#%d: MutexUnlock %zx all=%d\n", thr->tid, addr, all); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); + int rec = 0; if (s->recursion == 0) { if (!s->is_broken) { s->is_broken = true; - Printf("ThreadSanitizer WARNING: unlock of unlocked mutex\n"); + Printf("ThreadSanitizer WARNING: unlock of unlocked mutex %p\n", addr); PrintCurrentStack(thr, pc); } } else if (s->owner_tid != thr->tid) { if (!s->is_broken) { s->is_broken = true; - Printf("ThreadSanitizer WARNING: mutex unlock by another thread\n"); + Printf("ThreadSanitizer WARNING: mutex %p is unlocked by wrong thread\n", + addr); PrintCurrentStack(thr, pc); } } else { - s->recursion--; + rec = all ? s->recursion : 1; + s->recursion -= rec; if (s->recursion == 0) { StatInc(thr, StatMutexUnlock); s->owner_tid = SyncVar::kInvalidTid; @@ -145,6 +149,7 @@ void MutexUnlock(ThreadState *thr, uptr pc, uptr addr) { } thr->mset.Del(s->GetId(), true); s->mtx.Unlock(); + return rec; } void MutexReadLock(ThreadState *thr, uptr pc, uptr addr) { @@ -157,7 +162,8 @@ void MutexReadLock(ThreadState *thr, uptr pc, uptr addr) { thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId()); if (s->owner_tid != SyncVar::kInvalidTid) { - Printf("ThreadSanitizer WARNING: read lock of a write locked mutex\n"); + Printf("ThreadSanitizer WARNING: read lock of a write locked mutex %p\n", + addr); PrintCurrentStack(thr, pc); } thr->clock.set(thr->tid, thr->fast_state.epoch()); @@ -178,8 +184,8 @@ void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); if (s->owner_tid != SyncVar::kInvalidTid) { - Printf("ThreadSanitizer WARNING: read unlock of a write " - "locked mutex\n"); + Printf("ThreadSanitizer WARNING: read unlock of a write locked mutex %p\n", + addr); PrintCurrentStack(thr, pc); } thr->clock.set(thr->tid, thr->fast_state.epoch()); @@ -229,7 +235,8 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { } } else if (!s->is_broken) { s->is_broken = true; - Printf("ThreadSanitizer WARNING: mutex unlock by another thread\n"); + Printf("ThreadSanitizer WARNING: mutex %p is unlock by wrong thread\n", + addr); PrintCurrentStack(thr, pc); } thr->mset.Del(s->GetId(), write); @@ -246,18 +253,19 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) { s->mtx.ReadUnlock(); } +static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { + ThreadState *thr = reinterpret_cast<ThreadState*>(arg); + ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + if (tctx->status == ThreadStatusRunning) + thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch()); + else + thr->clock.set(tctx->tid, tctx->epoch1); +} + void AcquireGlobal(ThreadState *thr, uptr pc) { - Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0) - continue; - if (tctx->status == ThreadStatusRunning) - thr->clock.set(i, tctx->thr->fast_state.epoch()); - else - thr->clock.set(i, tctx->epoch1); - } + ThreadRegistryLock l(CTX()->thread_registry); + CTX()->thread_registry->RunCallbackForEachThreadLocked( + UpdateClockCallback, thr); } void Release(ThreadState *thr, uptr pc, uptr addr) { @@ -281,19 +289,20 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { } #ifndef TSAN_GO +static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { + ThreadState *thr = reinterpret_cast<ThreadState*>(arg); + ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + if (tctx->status == ThreadStatusRunning) + thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch()); + else + thr->last_sleep_clock.set(tctx->tid, tctx->epoch1); +} + void AfterSleep(ThreadState *thr, uptr pc) { - Context *ctx = CTX(); thr->last_sleep_stack_id = CurrentStackId(thr, pc); - Lock l(&ctx->thread_mtx); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0) - continue; - if (tctx->status == ThreadStatusRunning) - thr->last_sleep_clock.set(i, tctx->thr->fast_state.epoch()); - else - thr->last_sleep_clock.set(i, tctx->epoch1); - } + ThreadRegistryLock l(CTX()->thread_registry); + CTX()->thread_registry->RunCallbackForEachThreadLocked( + UpdateSleepClockCallback, thr); } #endif diff --git a/libsanitizer/tsan/tsan_rtl_report.cc b/libsanitizer/tsan/tsan_rtl_report.cc index ff1d43bc9e8..7c0a0280071 100644 --- a/libsanitizer/tsan/tsan_rtl_report.cc +++ b/libsanitizer/tsan/tsan_rtl_report.cc @@ -93,8 +93,9 @@ static void StackStripMain(ReportStack *stack) { DPrintf("Bottom stack frame of stack %zx is missed\n", stack->pc); } #else - if (last && 0 == internal_strcmp(last, "schedunlock")) - last_frame2->next = 0; + // The last frame always point into runtime (gosched0, goexit0, runtime.main). + last_frame2->next = 0; + (void)last; #endif } @@ -103,17 +104,25 @@ static ReportStack *SymbolizeStack(const StackTrace& trace) { return 0; ReportStack *stack = 0; for (uptr si = 0; si < trace.Size(); si++) { + const uptr pc = trace.Get(si); +#ifndef TSAN_GO // We obtain the return address, that is, address of the next instruction, // so offset it by 1 byte. - bool is_last = (si == trace.Size() - 1); - ReportStack *ent = SymbolizeCode(trace.Get(si) - !is_last); + const uptr pc1 = __sanitizer::StackTrace::GetPreviousInstructionPc(pc); +#else + // FIXME(dvyukov): Go sometimes uses address of a function as top pc. + uptr pc1 = pc; + if (si != trace.Size() - 1) + pc1 -= 1; +#endif + ReportStack *ent = SymbolizeCode(pc1); CHECK_NE(ent, 0); ReportStack *last = ent; while (last->next) { - last->pc += !is_last; + last->pc = pc; // restore original pc for report last = last->next; } - last->pc += !is_last; + last->pc = pc; // restore original pc for report last->next = stack; stack = ent; } @@ -123,14 +132,16 @@ static ReportStack *SymbolizeStack(const StackTrace& trace) { ScopedReport::ScopedReport(ReportType typ) { ctx_ = CTX(); - ctx_->thread_mtx.CheckLocked(); + ctx_->thread_registry->CheckLocked(); void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc)); rep_ = new(mem) ReportDesc; rep_->typ = typ; ctx_->report_mtx.Lock(); + CommonSanitizerReportMutex.Lock(); } ScopedReport::~ScopedReport() { + CommonSanitizerReportMutex.Unlock(); ctx_->report_mtx.Unlock(); DestroyAndFree(rep_); } @@ -175,7 +186,7 @@ void ScopedReport::AddMemoryAccess(uptr addr, Shadow s, void ScopedReport::AddThread(const ThreadContext *tctx) { for (uptr i = 0; i < rep_->threads.Size(); i++) { - if (rep_->threads[i]->id == tctx->tid) + if ((u32)rep_->threads[i]->id == tctx->tid) return; } void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread)); @@ -185,42 +196,65 @@ void ScopedReport::AddThread(const ThreadContext *tctx) { rt->pid = tctx->os_id; rt->running = (tctx->status == ThreadStatusRunning); rt->name = tctx->name ? internal_strdup(tctx->name) : 0; - rt->parent_tid = tctx->creation_tid; + rt->parent_tid = tctx->parent_tid; + rt->stack = 0; +#ifdef TSAN_GO rt->stack = SymbolizeStack(tctx->creation_stack); +#else + uptr ssz = 0; + const uptr *stack = StackDepotGet(tctx->creation_stack_id, &ssz); + if (stack) { + StackTrace trace; + trace.Init(stack, ssz); + rt->stack = SymbolizeStack(trace); + } +#endif } #ifndef TSAN_GO -static ThreadContext *FindThread(int unique_id) { +static ThreadContext *FindThreadByUidLocked(int unique_id) { Context *ctx = CTX(); - ctx->thread_mtx.CheckLocked(); + ctx->thread_registry->CheckLocked(); for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx && tctx->unique_id == unique_id) { + ThreadContext *tctx = static_cast<ThreadContext*>( + ctx->thread_registry->GetThreadLocked(i)); + if (tctx && tctx->unique_id == (u32)unique_id) { return tctx; } } return 0; } +static ThreadContext *FindThreadByTidLocked(int tid) { + Context *ctx = CTX(); + ctx->thread_registry->CheckLocked(); + return static_cast<ThreadContext*>( + ctx->thread_registry->GetThreadLocked(tid)); +} + +static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) { + uptr addr = (uptr)arg; + ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + if (tctx->status != ThreadStatusRunning) + return false; + ThreadState *thr = tctx->thr; + CHECK(thr); + return ((addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) || + (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size)); +} + ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { Context *ctx = CTX(); - ctx->thread_mtx.CheckLocked(); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0 || tctx->status != ThreadStatusRunning) - continue; - ThreadState *thr = tctx->thr; - CHECK(thr); - if (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) { - *is_stack = true; - return tctx; - } - if (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size) { - *is_stack = false; - return tctx; - } - } - return 0; + ctx->thread_registry->CheckLocked(); + ThreadContext *tctx = static_cast<ThreadContext*>( + ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls, + (void*)addr)); + if (!tctx) + return 0; + ThreadState *thr = tctx->thr; + CHECK(thr); + *is_stack = (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size); + return tctx; } #endif @@ -234,7 +268,16 @@ void ScopedReport::AddMutex(const SyncVar *s) { rep_->mutexes.PushBack(rm); rm->id = s->uid; rm->destroyed = false; - rm->stack = SymbolizeStack(s->creation_stack); + rm->stack = 0; +#ifndef TSAN_GO + uptr ssz = 0; + const uptr *stack = StackDepotGet(s->creation_stack_id, &ssz); + if (stack) { + StackTrace trace; + trace.Init(stack, ssz); + rm->stack = SymbolizeStack(trace); + } +#endif } void ScopedReport::AddMutex(u64 id) { @@ -272,27 +315,28 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { trace.Init(stack, ssz); loc->stack = SymbolizeStack(trace); } - ThreadContext *tctx = FindThread(creat_tid); + ThreadContext *tctx = FindThreadByUidLocked(creat_tid); if (tctx) AddThread(tctx); return; } - if (allocator()->PointerIsMine((void*)addr)) { - MBlock *b = user_mblock(0, (void*)addr); - ThreadContext *tctx = FindThread(b->alloc_tid); + MBlock *b = 0; + if (allocator()->PointerIsMine((void*)addr) + && (b = user_mblock(0, (void*)addr))) { + ThreadContext *tctx = FindThreadByTidLocked(b->Tid()); void *mem = internal_alloc(MBlockReportLoc, sizeof(ReportLocation)); ReportLocation *loc = new(mem) ReportLocation(); rep_->locs.PushBack(loc); loc->type = ReportLocationHeap; loc->addr = (uptr)allocator()->GetBlockBegin((void*)addr); - loc->size = b->size; - loc->tid = tctx ? tctx->tid : b->alloc_tid; + loc->size = b->Size(); + loc->tid = tctx ? tctx->tid : b->Tid(); loc->name = 0; loc->file = 0; loc->line = 0; loc->stack = 0; uptr ssz = 0; - const uptr *stack = StackDepotGet(b->alloc_stack_id, &ssz); + const uptr *stack = StackDepotGet(b->StackId(), &ssz); if (stack) { StackTrace trace; trace.Init(stack, ssz); @@ -331,6 +375,10 @@ void ScopedReport::AddSleep(u32 stack_id) { } #endif +void ScopedReport::SetCount(int count) { + rep_->count = count; +} + const ReportDesc *ScopedReport::GetReport() const { return rep_; } @@ -339,21 +387,17 @@ void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset) { // This function restores stack trace and mutex set for the thread/epoch. // It does so by getting stack trace and mutex set at the beginning of // trace part, and then replaying the trace till the given epoch. - ThreadContext *tctx = CTX()->threads[tid]; + Context *ctx = CTX(); + ctx->thread_registry->CheckLocked(); + ThreadContext *tctx = static_cast<ThreadContext*>( + ctx->thread_registry->GetThreadLocked(tid)); if (tctx == 0) return; - Trace* trace = 0; - if (tctx->status == ThreadStatusRunning) { - CHECK(tctx->thr); - trace = &tctx->thr->trace; - } else if (tctx->status == ThreadStatusFinished - || tctx->status == ThreadStatusDead) { - if (tctx->dead_info == 0) - return; - trace = &tctx->dead_info->trace; - } else { + if (tctx->status != ThreadStatusRunning + && tctx->status != ThreadStatusFinished + && tctx->status != ThreadStatusDead) return; - } + Trace* trace = ThreadTrace(tctx->tid); Lock l(&trace->mtx); const int partidx = (epoch / kTracePartSize) % TraceParts(); TraceHeader* hdr = &trace->headers[partidx]; @@ -464,31 +508,58 @@ static void AddRacyStacks(ThreadState *thr, const StackTrace (&traces)[2], bool OutputReport(Context *ctx, const ScopedReport &srep, const ReportStack *suppress_stack1, - const ReportStack *suppress_stack2) { + const ReportStack *suppress_stack2, + const ReportLocation *suppress_loc) { + atomic_store(&ctx->last_symbolize_time_ns, NanoTime(), memory_order_relaxed); const ReportDesc *rep = srep.GetReport(); - uptr suppress_pc = IsSuppressed(rep->typ, suppress_stack1); + Suppression *supp = 0; + uptr suppress_pc = IsSuppressed(rep->typ, suppress_stack1, &supp); + if (suppress_pc == 0) + suppress_pc = IsSuppressed(rep->typ, suppress_stack2, &supp); if (suppress_pc == 0) - suppress_pc = IsSuppressed(rep->typ, suppress_stack2); + suppress_pc = IsSuppressed(rep->typ, suppress_loc, &supp); if (suppress_pc != 0) { - FiredSuppression supp = {srep.GetReport()->typ, suppress_pc}; - ctx->fired_suppressions.PushBack(supp); + FiredSuppression s = {srep.GetReport()->typ, suppress_pc, supp}; + ctx->fired_suppressions.push_back(s); } if (OnReport(rep, suppress_pc != 0)) return false; PrintReport(rep); - CTX()->nreported++; + ctx->nreported++; + if (flags()->halt_on_error) + internal__exit(flags()->exitcode); return true; } bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, const StackTrace &trace) { - for (uptr k = 0; k < ctx->fired_suppressions.Size(); k++) { + for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { if (ctx->fired_suppressions[k].type != srep.GetReport()->typ) continue; for (uptr j = 0; j < trace.Size(); j++) { - if (trace.Get(j) == ctx->fired_suppressions[k].pc) + FiredSuppression *s = &ctx->fired_suppressions[k]; + if (trace.Get(j) == s->pc) { + if (s->supp) + s->supp->hit_count++; return true; + } + } + } + return false; +} + +static bool IsFiredSuppression(Context *ctx, + const ScopedReport &srep, + uptr addr) { + for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { + if (ctx->fired_suppressions[k].type != srep.GetReport()->typ) + continue; + FiredSuppression *s = &ctx->fired_suppressions[k]; + if (addr == s->pc) { + if (s->supp) + s->supp->hit_count++; + return true; } } return false; @@ -525,8 +596,8 @@ static bool IsJavaNonsense(const ReportDesc *rep) { || (frame->func == 0 && frame->file == 0 && frame->line == 0 && frame->module == 0)) { if (frame) { - FiredSuppression supp = {rep->typ, frame->pc}; - CTX()->fired_suppressions.PushBack(supp); + FiredSuppression supp = {rep->typ, frame->pc, 0}; + CTX()->fired_suppressions.push_back(supp); } return true; } @@ -557,10 +628,6 @@ void ReportRace(ThreadState *thr) { if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr)) return; - if (thr->in_signal_handler) - Printf("ThreadSanitizer: printing report from signal handler." - " Can crash or hang.\n"); - bool freed = false; { Shadow s(thr->racy_state[1]); @@ -583,9 +650,16 @@ void ReportRace(ThreadState *thr) { } Context *ctx = CTX(); - Lock l0(&ctx->thread_mtx); - - ScopedReport rep(freed ? ReportTypeUseAfterFree : ReportTypeRace); + ThreadRegistryLock l0(ctx->thread_registry); + + ReportType typ = ReportTypeRace; + if (thr->is_vptr_access) + typ = ReportTypeVptrRace; + else if (freed) + typ = ReportTypeUseAfterFree; + ScopedReport rep(typ); + if (IsFiredSuppression(ctx, rep, addr)) + return; const uptr kMop = 2; StackTrace traces[kMop]; const uptr toppc = TraceTopPC(thr); @@ -596,6 +670,8 @@ void ReportRace(ThreadState *thr) { new(mset2.data()) MutexSet(); Shadow s2(thr->racy_state[1]); RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2.data()); + if (IsFiredSuppression(ctx, rep, traces[1])) + return; if (HandleRacyStacks(thr, traces, addr_min, addr_max)) return; @@ -611,7 +687,8 @@ void ReportRace(ThreadState *thr) { for (uptr i = 0; i < kMop; i++) { FastState s(thr->racy_state[i]); - ThreadContext *tctx = ctx->threads[s.tid()]; + ThreadContext *tctx = static_cast<ThreadContext*>( + ctx->thread_registry->GetThreadLocked(s.tid())); if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1) continue; rep.AddThread(tctx); @@ -627,8 +704,11 @@ void ReportRace(ThreadState *thr) { } #endif + ReportLocation *suppress_loc = rep.GetReport()->locs.Size() ? + rep.GetReport()->locs[0] : 0; if (!OutputReport(ctx, rep, rep.GetReport()->mops[0]->stack, - rep.GetReport()->mops[1]->stack)) + rep.GetReport()->mops[1]->stack, + suppress_loc)) return; AddRacyStacks(thr, traces, addr_min, addr_max); @@ -646,6 +726,11 @@ void PrintCurrentStackSlow() { sizeof(__sanitizer::StackTrace))) __sanitizer::StackTrace; ptrace->SlowUnwindStack(__sanitizer::StackTrace::GetCurrentPc(), kStackTraceMax); + for (uptr i = 0; i < ptrace->size / 2; i++) { + uptr tmp = ptrace->trace[i]; + ptrace->trace[i] = ptrace->trace[ptrace->size - i - 1]; + ptrace->trace[ptrace->size - i - 1] = tmp; + } StackTrace trace; trace.Init(ptrace->trace, ptrace->size); PrintStack(SymbolizeStack(trace)); diff --git a/libsanitizer/tsan/tsan_rtl_thread.cc b/libsanitizer/tsan/tsan_rtl_thread.cc index e30916dc627..9811e1c015f 100644 --- a/libsanitizer/tsan/tsan_rtl_thread.cc +++ b/libsanitizer/tsan/tsan_rtl_thread.cc @@ -18,144 +18,193 @@ namespace __tsan { +// ThreadContext implementation. + +ThreadContext::ThreadContext(int tid) + : ThreadContextBase(tid) + , thr() + , sync() + , epoch0() + , epoch1() { +} + #ifndef TSAN_GO -const int kThreadQuarantineSize = 16; -#else -const int kThreadQuarantineSize = 64; +ThreadContext::~ThreadContext() { +} #endif -static void MaybeReportThreadLeak(ThreadContext *tctx) { - if (tctx->detached) +void ThreadContext::OnDead() { + sync.Reset(); +} + +void ThreadContext::OnJoined(void *arg) { + ThreadState *caller_thr = static_cast<ThreadState *>(arg); + caller_thr->clock.acquire(&sync); + StatInc(caller_thr, StatSyncAcquire); + sync.Reset(); +} + +struct OnCreatedArgs { + ThreadState *thr; + uptr pc; +}; + +void ThreadContext::OnCreated(void *arg) { + thr = 0; + if (tid == 0) return; - if (tctx->status != ThreadStatusCreated - && tctx->status != ThreadStatusRunning - && tctx->status != ThreadStatusFinished) + OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg); + args->thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); + args->thr->clock.set(args->thr->tid, args->thr->fast_state.epoch()); + args->thr->fast_synch_epoch = args->thr->fast_state.epoch(); + args->thr->clock.release(&sync); + StatInc(args->thr, StatSyncRelease); +#ifdef TSAN_GO + creation_stack.ObtainCurrent(args->thr, args->pc); +#else + creation_stack_id = CurrentStackId(args->thr, args->pc); +#endif + if (reuse_count == 0) + StatInc(args->thr, StatThreadMaxTid); +} + +void ThreadContext::OnReset() { + sync.Reset(); + FlushUnneededShadowMemory(GetThreadTrace(tid), TraceSize() * sizeof(Event)); + //!!! FlushUnneededShadowMemory(GetThreadTraceHeader(tid), sizeof(Trace)); +} + +struct OnStartedArgs { + ThreadState *thr; + uptr stk_addr; + uptr stk_size; + uptr tls_addr; + uptr tls_size; +}; + +void ThreadContext::OnStarted(void *arg) { + OnStartedArgs *args = static_cast<OnStartedArgs*>(arg); + thr = args->thr; + // RoundUp so that one trace part does not contain events + // from different threads. + epoch0 = RoundUp(epoch1 + 1, kTracePartSize); + epoch1 = (u64)-1; + new(thr) ThreadState(CTX(), tid, unique_id, + epoch0, args->stk_addr, args->stk_size, args->tls_addr, args->tls_size); +#ifdef TSAN_GO + // Setup dynamic shadow stack. + const int kInitStackSize = 8; + args->thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack, + kInitStackSize * sizeof(uptr)); + args->thr->shadow_stack_pos = thr->shadow_stack; + args->thr->shadow_stack_end = thr->shadow_stack + kInitStackSize; +#endif +#ifndef TSAN_GO + AllocatorThreadStart(args->thr); +#endif + thr = args->thr; + thr->fast_synch_epoch = epoch0; + thr->clock.set(tid, epoch0); + thr->clock.acquire(&sync); + thr->fast_state.SetHistorySize(flags()->history_size); + const uptr trace = (epoch0 / kTracePartSize) % TraceParts(); + Trace *thr_trace = ThreadTrace(thr->tid); + thr_trace->headers[trace].epoch0 = epoch0; + StatInc(thr, StatSyncAcquire); + sync.Reset(); + DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " + "tls_addr=%zx tls_size=%zx\n", + tid, (uptr)epoch0, args->stk_addr, args->stk_size, + args->tls_addr, args->tls_size); + thr->is_alive = true; +} + +void ThreadContext::OnFinished() { + if (!detached) { + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + thr->clock.set(thr->tid, thr->fast_state.epoch()); + thr->fast_synch_epoch = thr->fast_state.epoch(); + thr->clock.release(&sync); + StatInc(thr, StatSyncRelease); + } + epoch1 = thr->fast_state.epoch(); + +#ifndef TSAN_GO + AllocatorThreadFinish(thr); +#endif + thr->~ThreadState(); + StatAggregate(CTX()->stat, thr->stat); + thr = 0; +} + +#ifndef TSAN_GO +struct ThreadLeak { + ThreadContext *tctx; + int count; +}; + +static void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) { + Vector<ThreadLeak> &leaks = *(Vector<ThreadLeak>*)arg; + ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + if (tctx->detached || tctx->status != ThreadStatusFinished) return; - ScopedReport rep(ReportTypeThreadLeak); - rep.AddThread(tctx); - OutputReport(CTX(), rep); + for (uptr i = 0; i < leaks.Size(); i++) { + if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) { + leaks[i].count++; + return; + } + } + ThreadLeak leak = {tctx, 1}; + leaks.PushBack(leak); +} +#endif + +static void ThreadCheckIgnore(ThreadState *thr) { + if (thr->ignore_reads_and_writes) { + Printf("ThreadSanitizer: thread T%d finished with ignores enabled.\n", + thr->tid); + } } void ThreadFinalize(ThreadState *thr) { CHECK_GT(thr->in_rtl, 0); + ThreadCheckIgnore(thr); +#ifndef TSAN_GO if (!flags()->report_thread_leaks) return; - Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0) - continue; - MaybeReportThreadLeak(tctx); + ThreadRegistryLock l(CTX()->thread_registry); + Vector<ThreadLeak> leaks(MBlockScopedBuf); + CTX()->thread_registry->RunCallbackForEachThreadLocked( + MaybeReportThreadLeak, &leaks); + for (uptr i = 0; i < leaks.Size(); i++) { + ScopedReport rep(ReportTypeThreadLeak); + rep.AddThread(leaks[i].tctx); + rep.SetCount(leaks[i].count); + OutputReport(CTX(), rep); } +#endif } int ThreadCount(ThreadState *thr) { CHECK_GT(thr->in_rtl, 0); Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - int cnt = 0; - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = ctx->threads[i]; - if (tctx == 0) - continue; - if (tctx->status != ThreadStatusCreated - && tctx->status != ThreadStatusRunning) - continue; - cnt++; - } - return cnt; -} - -static void ThreadDead(ThreadState *thr, ThreadContext *tctx) { - Context *ctx = CTX(); - CHECK_GT(thr->in_rtl, 0); - CHECK(tctx->status == ThreadStatusRunning - || tctx->status == ThreadStatusFinished); - DPrintf("#%d: ThreadDead uid=%zu\n", thr->tid, tctx->user_id); - tctx->status = ThreadStatusDead; - tctx->user_id = 0; - tctx->sync.Reset(); - - // Put to dead list. - tctx->dead_next = 0; - if (ctx->dead_list_size == 0) - ctx->dead_list_head = tctx; - else - ctx->dead_list_tail->dead_next = tctx; - ctx->dead_list_tail = tctx; - ctx->dead_list_size++; + uptr result; + ctx->thread_registry->GetNumberOfThreads(0, 0, &result); + return (int)result; } int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { CHECK_GT(thr->in_rtl, 0); - Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); StatInc(thr, StatThreadCreate); - int tid = -1; - ThreadContext *tctx = 0; - if (ctx->dead_list_size > kThreadQuarantineSize - || ctx->thread_seq >= kMaxTid) { - // Reusing old thread descriptor and tid. - if (ctx->dead_list_size == 0) { - Printf("ThreadSanitizer: %d thread limit exceeded. Dying.\n", - kMaxTid); - Die(); - } - StatInc(thr, StatThreadReuse); - tctx = ctx->dead_list_head; - ctx->dead_list_head = tctx->dead_next; - ctx->dead_list_size--; - if (ctx->dead_list_size == 0) { - CHECK_EQ(tctx->dead_next, 0); - ctx->dead_list_head = 0; - } - CHECK_EQ(tctx->status, ThreadStatusDead); - tctx->status = ThreadStatusInvalid; - tctx->reuse_count++; - tctx->sync.Reset(); - tid = tctx->tid; - DestroyAndFree(tctx->dead_info); - if (tctx->name) { - internal_free(tctx->name); - tctx->name = 0; - } - } else { - // Allocating new thread descriptor and tid. - StatInc(thr, StatThreadMaxTid); - tid = ctx->thread_seq++; - void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); - tctx = new(mem) ThreadContext(tid); - ctx->threads[tid] = tctx; - MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event)); - } - CHECK_NE(tctx, 0); - CHECK_GE(tid, 0); - CHECK_LT(tid, kMaxTid); + Context *ctx = CTX(); + OnCreatedArgs args = { thr, pc }; + int tid = ctx->thread_registry->CreateThread(uid, detached, thr->tid, &args); DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", thr->tid, tid, uid); - CHECK_EQ(tctx->status, ThreadStatusInvalid); - ctx->alive_threads++; - if (ctx->max_alive_threads < ctx->alive_threads) { - ctx->max_alive_threads++; - CHECK_EQ(ctx->max_alive_threads, ctx->alive_threads); - StatInc(thr, StatThreadMaxAlive); - } - tctx->status = ThreadStatusCreated; - tctx->thr = 0; - tctx->user_id = uid; - tctx->unique_id = ctx->unique_thread_seq++; - tctx->detached = detached; - if (tid) { - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - thr->clock.set(thr->tid, thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.release(&tctx->sync); - StatInc(thr, StatSyncRelease); - tctx->creation_stack.ObtainCurrent(thr, pc); - tctx->creation_tid = thr->tid; - } + StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads()); return tid; } @@ -168,9 +217,8 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { - if (stk_addr && stk_size) { - MemoryResetRange(thr, /*pc=*/ 1, stk_addr, stk_size); - } + if (stk_addr && stk_size) + MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size); if (tls_addr && tls_size) { // Check that the thr object is in tls; @@ -181,116 +229,42 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { CHECK_GE(thr_end, tls_addr); CHECK_LE(thr_end, tls_addr + tls_size); // Since the thr object is huge, skip it. - MemoryResetRange(thr, /*pc=*/ 2, tls_addr, thr_beg - tls_addr); - MemoryResetRange(thr, /*pc=*/ 2, thr_end, tls_addr + tls_size - thr_end); + MemoryRangeImitateWrite(thr, /*pc=*/ 2, tls_addr, thr_beg - tls_addr); + MemoryRangeImitateWrite(thr, /*pc=*/ 2, + thr_end, tls_addr + tls_size - thr_end); } } - Lock l(&CTX()->thread_mtx); - ThreadContext *tctx = CTX()->threads[tid]; - CHECK_NE(tctx, 0); - CHECK_EQ(tctx->status, ThreadStatusCreated); - tctx->status = ThreadStatusRunning; - tctx->os_id = os_id; - // RoundUp so that one trace part does not contain events - // from different threads. - tctx->epoch0 = RoundUp(tctx->epoch1 + 1, kTracePartSize); - tctx->epoch1 = (u64)-1; - new(thr) ThreadState(CTX(), tid, tctx->unique_id, - tctx->epoch0, stk_addr, stk_size, - tls_addr, tls_size); -#ifdef TSAN_GO - // Setup dynamic shadow stack. - const int kInitStackSize = 8; - thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack, - kInitStackSize * sizeof(uptr)); - thr->shadow_stack_pos = thr->shadow_stack; - thr->shadow_stack_end = thr->shadow_stack + kInitStackSize; -#endif -#ifndef TSAN_GO - AllocatorThreadStart(thr); -#endif - tctx->thr = thr; - thr->fast_synch_epoch = tctx->epoch0; - thr->clock.set(tid, tctx->epoch0); - thr->clock.acquire(&tctx->sync); - thr->fast_state.SetHistorySize(flags()->history_size); - const uptr trace = (tctx->epoch0 / kTracePartSize) % TraceParts(); - thr->trace.headers[trace].epoch0 = tctx->epoch0; - StatInc(thr, StatSyncAcquire); - DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " - "tls_addr=%zx tls_size=%zx\n", - tid, (uptr)tctx->epoch0, stk_addr, stk_size, tls_addr, tls_size); - thr->is_alive = true; + OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; + CTX()->thread_registry->StartThread(tid, os_id, &args); } void ThreadFinish(ThreadState *thr) { CHECK_GT(thr->in_rtl, 0); + ThreadCheckIgnore(thr); StatInc(thr, StatThreadFinish); - // FIXME: Treat it as write. if (thr->stk_addr && thr->stk_size) - MemoryResetRange(thr, /*pc=*/ 3, thr->stk_addr, thr->stk_size); - if (thr->tls_addr && thr->tls_size) { - const uptr thr_beg = (uptr)thr; - const uptr thr_end = (uptr)thr + sizeof(*thr); - // Since the thr object is huge, skip it. - MemoryResetRange(thr, /*pc=*/ 4, thr->tls_addr, thr_beg - thr->tls_addr); - MemoryResetRange(thr, /*pc=*/ 5, - thr_end, thr->tls_addr + thr->tls_size - thr_end); - } + DontNeedShadowFor(thr->stk_addr, thr->stk_size); + if (thr->tls_addr && thr->tls_size) + DontNeedShadowFor(thr->tls_addr, thr->tls_size); thr->is_alive = false; Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - ThreadContext *tctx = ctx->threads[thr->tid]; - CHECK_NE(tctx, 0); - CHECK_EQ(tctx->status, ThreadStatusRunning); - CHECK_GT(ctx->alive_threads, 0); - ctx->alive_threads--; - if (tctx->detached) { - ThreadDead(thr, tctx); - } else { - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - thr->clock.set(thr->tid, thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.release(&tctx->sync); - StatInc(thr, StatSyncRelease); - tctx->status = ThreadStatusFinished; - } + ctx->thread_registry->FinishThread(thr->tid); +} - // Save from info about the thread. - tctx->dead_info = new(internal_alloc(MBlockDeadInfo, sizeof(ThreadDeadInfo))) - ThreadDeadInfo(); - for (uptr i = 0; i < TraceParts(); i++) { - tctx->dead_info->trace.headers[i].epoch0 = thr->trace.headers[i].epoch0; - tctx->dead_info->trace.headers[i].stack0.CopyFrom( - thr->trace.headers[i].stack0); +static bool FindThreadByUid(ThreadContextBase *tctx, void *arg) { + uptr uid = (uptr)arg; + if (tctx->user_id == uid && tctx->status != ThreadStatusInvalid) { + tctx->user_id = 0; + return true; } - tctx->epoch1 = thr->fast_state.epoch(); - -#ifndef TSAN_GO - AllocatorThreadFinish(thr); -#endif - thr->~ThreadState(); - StatAggregate(ctx->stat, thr->stat); - tctx->thr = 0; + return false; } int ThreadTid(ThreadState *thr, uptr pc, uptr uid) { CHECK_GT(thr->in_rtl, 0); Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - int res = -1; - for (unsigned tid = 0; tid < kMaxTid; tid++) { - ThreadContext *tctx = ctx->threads[tid]; - if (tctx != 0 && tctx->user_id == uid - && tctx->status != ThreadStatusInvalid) { - tctx->user_id = 0; - res = tid; - break; - } - } + int res = ctx->thread_registry->FindThread(FindThreadByUid, (void*)uid); DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, res); return res; } @@ -301,18 +275,7 @@ void ThreadJoin(ThreadState *thr, uptr pc, int tid) { CHECK_LT(tid, kMaxTid); DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid); Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - ThreadContext *tctx = ctx->threads[tid]; - if (tctx->status == ThreadStatusInvalid) { - Printf("ThreadSanitizer: join of non-existent thread\n"); - return; - } - // FIXME(dvyukov): print message and continue (it's user error). - CHECK_EQ(tctx->detached, false); - CHECK_EQ(tctx->status, ThreadStatusFinished); - thr->clock.acquire(&tctx->sync); - StatInc(thr, StatSyncAcquire); - ThreadDead(thr, tctx); + ctx->thread_registry->JoinThread(tid, thr); } void ThreadDetach(ThreadState *thr, uptr pc, int tid) { @@ -320,31 +283,12 @@ void ThreadDetach(ThreadState *thr, uptr pc, int tid) { CHECK_GT(tid, 0); CHECK_LT(tid, kMaxTid); Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - ThreadContext *tctx = ctx->threads[tid]; - if (tctx->status == ThreadStatusInvalid) { - Printf("ThreadSanitizer: detach of non-existent thread\n"); - return; - } - if (tctx->status == ThreadStatusFinished) { - ThreadDead(thr, tctx); - } else { - tctx->detached = true; - } + ctx->thread_registry->DetachThread(tid); } void ThreadSetName(ThreadState *thr, const char *name) { - Context *ctx = CTX(); - Lock l(&ctx->thread_mtx); - ThreadContext *tctx = ctx->threads[thr->tid]; - CHECK_NE(tctx, 0); - CHECK_EQ(tctx->status, ThreadStatusRunning); - if (tctx->name) { - internal_free(tctx->name); - tctx->name = 0; - } - if (name) - tctx->name = internal_strdup(name); + CHECK_GT(thr->in_rtl, 0); + CTX()->thread_registry->SetThreadName(thr->tid, name); } void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, @@ -379,6 +323,13 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, StatInc(thr, StatMopRange); + if (*shadow_mem == kShadowRodata) { + // Access to .rodata section, no races here. + // Measurements show that it can be 10-20% of all memory accesses. + StatInc(thr, StatMopRangeRodata); + return; + } + FastState fast_state = thr->fast_state; if (fast_state.GetIgnoreBit()) return; @@ -421,25 +372,4 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } } -void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr, - uptr size, uptr step, bool is_write) { - if (size == 0) - return; - FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) - return; - StatInc(thr, StatMopRange); - fast_state.IncrementEpoch(); - thr->fast_state = fast_state; - TraceAddEvent(thr, fast_state, EventTypeMop, pc); - - for (uptr addr_end = addr + size; addr < addr_end; addr += step) { - u64 *shadow_mem = (u64*)MemToShadow(addr); - Shadow cur(fast_state); - cur.SetWrite(is_write); - cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kSizeLog1); - MemoryAccessImpl(thr, addr, kSizeLog1, is_write, false, - shadow_mem, cur); - } -} } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_stat.cc b/libsanitizer/tsan/tsan_stat.cc index fbec4225d9c..3a3ac1172d8 100644 --- a/libsanitizer/tsan/tsan_stat.cc +++ b/libsanitizer/tsan/tsan_stat.cc @@ -36,6 +36,8 @@ void StatOutput(u64 *stat) { name[StatMop8] = " size 8 "; name[StatMopSame] = " Including same "; name[StatMopRange] = " Including range "; + name[StatMopRodata] = " Including .rodata "; + name[StatMopRangeRodata] = " Including .rodata range "; name[StatShadowProcessed] = "Shadow processed "; name[StatShadowZero] = " Including empty "; name[StatShadowNonZero] = " Including non empty "; @@ -103,6 +105,7 @@ void StatOutput(u64 *stat) { name[StatInt_realloc] = " realloc "; name[StatInt_free] = " free "; name[StatInt_cfree] = " cfree "; + name[StatInt_malloc_usable_size] = " malloc_usable_size "; name[StatInt_mmap] = " mmap "; name[StatInt_mmap64] = " mmap64 "; name[StatInt_munmap] = " munmap "; @@ -133,6 +136,9 @@ void StatOutput(u64 *stat) { name[StatInt_strcpy] = " strcpy "; name[StatInt_strncpy] = " strncpy "; name[StatInt_strstr] = " strstr "; + name[StatInt_strdup] = " strdup "; + name[StatInt_strcasecmp] = " strcasecmp "; + name[StatInt_strncasecmp] = " strncasecmp "; name[StatInt_atexit] = " atexit "; name[StatInt___cxa_guard_acquire] = " __cxa_guard_acquire "; name[StatInt___cxa_guard_release] = " __cxa_guard_release "; @@ -172,6 +178,7 @@ void StatOutput(u64 *stat) { name[StatInt_pthread_barrier_destroy] = " pthread_barrier_destroy "; name[StatInt_pthread_barrier_wait] = " pthread_barrier_wait "; name[StatInt_pthread_once] = " pthread_once "; + name[StatInt_pthread_getschedparam] = " pthread_getschedparam "; name[StatInt_sem_init] = " sem_init "; name[StatInt_sem_destroy] = " sem_destroy "; name[StatInt_sem_wait] = " sem_wait "; @@ -221,11 +228,13 @@ void StatOutput(u64 *stat) { name[StatInt_pread] = " pread "; name[StatInt_pread64] = " pread64 "; name[StatInt_readv] = " readv "; + name[StatInt_preadv] = " preadv "; name[StatInt_preadv64] = " preadv64 "; name[StatInt_write] = " write "; name[StatInt_pwrite] = " pwrite "; name[StatInt_pwrite64] = " pwrite64 "; name[StatInt_writev] = " writev "; + name[StatInt_pwritev] = " pwritev "; name[StatInt_pwritev64] = " pwritev64 "; name[StatInt_send] = " send "; name[StatInt_sendmsg] = " sendmsg "; @@ -237,13 +246,21 @@ void StatOutput(u64 *stat) { name[StatInt_fclose] = " fclose "; name[StatInt_fread] = " fread "; name[StatInt_fwrite] = " fwrite "; + name[StatInt_fflush] = " fflush "; + name[StatInt_abort] = " abort "; name[StatInt_puts] = " puts "; name[StatInt_rmdir] = " rmdir "; name[StatInt_opendir] = " opendir "; name[StatInt_epoll_ctl] = " epoll_ctl "; name[StatInt_epoll_wait] = " epoll_wait "; name[StatInt_poll] = " poll "; + name[StatInt_ppoll] = " ppoll "; name[StatInt_sigaction] = " sigaction "; + name[StatInt_signal] = " signal "; + name[StatInt_sigsuspend] = " sigsuspend "; + name[StatInt_raise] = " raise "; + name[StatInt_kill] = " kill "; + name[StatInt_pthread_kill] = " pthread_kill "; name[StatInt_sleep] = " sleep "; name[StatInt_usleep] = " usleep "; name[StatInt_nanosleep] = " nanosleep "; @@ -271,6 +288,87 @@ void StatOutput(u64 *stat) { name[StatInt_ctime_r] = " ctime_r "; name[StatInt_asctime] = " asctime "; name[StatInt_asctime_r] = " asctime_r "; + name[StatInt_frexp] = " frexp "; + name[StatInt_frexpf] = " frexpf "; + name[StatInt_frexpl] = " frexpl "; + name[StatInt_getpwnam] = " getpwnam "; + name[StatInt_getpwuid] = " getpwuid "; + name[StatInt_getgrnam] = " getgrnam "; + name[StatInt_getgrgid] = " getgrgid "; + name[StatInt_getpwnam_r] = " getpwnam_r "; + name[StatInt_getpwuid_r] = " getpwuid_r "; + name[StatInt_getgrnam_r] = " getgrnam_r "; + name[StatInt_getgrgid_r] = " getgrgid_r "; + name[StatInt_clock_getres] = " clock_getres "; + name[StatInt_clock_gettime] = " clock_gettime "; + name[StatInt_clock_settime] = " clock_settime "; + name[StatInt_getitimer] = " getitimer "; + name[StatInt_setitimer] = " setitimer "; + name[StatInt_time] = " time "; + name[StatInt_glob] = " glob "; + name[StatInt_glob64] = " glob64 "; + name[StatInt_wait] = " wait "; + name[StatInt_waitid] = " waitid "; + name[StatInt_waitpid] = " waitpid "; + name[StatInt_wait3] = " wait3 "; + name[StatInt_wait4] = " wait4 "; + name[StatInt_inet_ntop] = " inet_ntop "; + name[StatInt_inet_pton] = " inet_pton "; + name[StatInt_inet_aton] = " inet_aton "; + name[StatInt_getaddrinfo] = " getaddrinfo "; + name[StatInt_getnameinfo] = " getnameinfo "; + name[StatInt_getsockname] = " getsockname "; + name[StatInt_gethostent] = " gethostent "; + name[StatInt_gethostbyname] = " gethostbyname "; + name[StatInt_gethostbyname2] = " gethostbyname2 "; + name[StatInt_gethostbyaddr] = " gethostbyaddr "; + name[StatInt_gethostent_r] = " gethostent_r "; + name[StatInt_gethostbyname_r] = " gethostbyname_r "; + name[StatInt_gethostbyname2_r] = " gethostbyname2_r "; + name[StatInt_gethostbyaddr_r] = " gethostbyaddr_r "; + name[StatInt_getsockopt] = " getsockopt "; + name[StatInt_modf] = " modf "; + name[StatInt_modff] = " modff "; + name[StatInt_modfl] = " modfl "; + name[StatInt_getpeername] = " getpeername "; + name[StatInt_ioctl] = " ioctl "; + name[StatInt_sysinfo] = " sysinfo "; + name[StatInt_readdir] = " readdir "; + name[StatInt_readdir64] = " readdir64 "; + name[StatInt_readdir_r] = " readdir_r "; + name[StatInt_readdir64_r] = " readdir64_r "; + name[StatInt_ptrace] = " ptrace "; + name[StatInt_setlocale] = " setlocale "; + name[StatInt_getcwd] = " getcwd "; + name[StatInt_get_current_dir_name] = " get_current_dir_name "; + name[StatInt_strtoimax] = " strtoimax "; + name[StatInt_strtoumax] = " strtoumax "; + name[StatInt_mbstowcs] = " mbstowcs "; + name[StatInt_mbsrtowcs] = " mbsrtowcs "; + name[StatInt_mbsnrtowcs] = " mbsnrtowcs "; + name[StatInt_wcstombs] = " wcstombs "; + name[StatInt_wcsrtombs] = " wcsrtombs "; + name[StatInt_wcsnrtombs] = " wcsnrtombs "; + name[StatInt_tcgetattr] = " tcgetattr "; + name[StatInt_realpath] = " realpath "; + name[StatInt_canonicalize_file_name] = " canonicalize_file_name "; + name[StatInt_confstr] = " confstr "; + name[StatInt_sched_getaffinity] = " sched_getaffinity "; + name[StatInt_strerror] = " strerror "; + name[StatInt_strerror_r] = " strerror_r "; + name[StatInt_scandir] = " scandir "; + name[StatInt_scandir64] = " scandir64 "; + name[StatInt_getgroups] = " getgroups "; + name[StatInt_wordexp] = " wordexp "; + name[StatInt_sigwait] = " sigwait "; + name[StatInt_sigwaitinfo] = " sigwaitinfo "; + name[StatInt_sigtimedwait] = " sigtimedwait "; + name[StatInt_sigemptyset] = " sigemptyset "; + name[StatInt_sigfillset] = " sigfillset "; + name[StatInt_sigpending] = " sigpending "; + name[StatInt_sigprocmask] = " sigprocmask "; + name[StatInt_backtrace] = " backtrace "; + name[StatInt_backtrace_symbols] = " backtrace_symbols "; name[StatAnnotation] = "Dynamic annotations "; name[StatAnnotateHappensBefore] = " HappensBefore "; @@ -280,6 +378,7 @@ void StatOutput(u64 *stat) { name[StatAnnotateMutexIsNotPHB] = " MutexIsNotPHB "; name[StatAnnotateCondVarWait] = " CondVarWait "; name[StatAnnotateRWLockCreate] = " RWLockCreate "; + name[StatAnnotateRWLockCreateStatic] = " StatAnnotateRWLockCreateStatic "; name[StatAnnotateRWLockDestroy] = " RWLockDestroy "; name[StatAnnotateRWLockAcquired] = " RWLockAcquired "; name[StatAnnotateRWLockReleased] = " RWLockReleased "; diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h index 8b08a024be2..e392ff62c63 100644 --- a/libsanitizer/tsan/tsan_stat.h +++ b/libsanitizer/tsan/tsan_stat.h @@ -25,6 +25,8 @@ enum StatType { StatMop8, StatMopSame, StatMopRange, + StatMopRodata, + StatMopRangeRodata, StatShadowProcessed, StatShadowZero, StatShadowNonZero, // Derived. @@ -100,6 +102,7 @@ enum StatType { StatInt_realloc, StatInt_free, StatInt_cfree, + StatInt_malloc_usable_size, StatInt_mmap, StatInt_mmap64, StatInt_munmap, @@ -129,7 +132,10 @@ enum StatType { StatInt_strncmp, StatInt_strcpy, StatInt_strncpy, + StatInt_strcasecmp, + StatInt_strncasecmp, StatInt_strstr, + StatInt_strdup, StatInt_atexit, StatInt___cxa_guard_acquire, StatInt___cxa_guard_release, @@ -167,6 +173,7 @@ enum StatType { StatInt_pthread_barrier_destroy, StatInt_pthread_barrier_wait, StatInt_pthread_once, + StatInt_pthread_getschedparam, StatInt_sem_init, StatInt_sem_destroy, StatInt_sem_wait, @@ -216,11 +223,13 @@ enum StatType { StatInt_pread, StatInt_pread64, StatInt_readv, + StatInt_preadv, StatInt_preadv64, StatInt_write, StatInt_pwrite, StatInt_pwrite64, StatInt_writev, + StatInt_pwritev, StatInt_pwritev64, StatInt_send, StatInt_sendmsg, @@ -232,14 +241,18 @@ enum StatType { StatInt_fclose, StatInt_fread, StatInt_fwrite, + StatInt_fflush, + StatInt_abort, StatInt_puts, StatInt_rmdir, StatInt_opendir, StatInt_epoll_ctl, StatInt_epoll_wait, StatInt_poll, + StatInt_ppoll, StatInt_sigaction, StatInt_signal, + StatInt_sigsuspend, StatInt_raise, StatInt_kill, StatInt_pthread_kill, @@ -270,6 +283,87 @@ enum StatType { StatInt_ctime_r, StatInt_asctime, StatInt_asctime_r, + StatInt_frexp, + StatInt_frexpf, + StatInt_frexpl, + StatInt_getpwnam, + StatInt_getpwuid, + StatInt_getgrnam, + StatInt_getgrgid, + StatInt_getpwnam_r, + StatInt_getpwuid_r, + StatInt_getgrnam_r, + StatInt_getgrgid_r, + StatInt_clock_getres, + StatInt_clock_gettime, + StatInt_clock_settime, + StatInt_getitimer, + StatInt_setitimer, + StatInt_time, + StatInt_glob, + StatInt_glob64, + StatInt_wait, + StatInt_waitid, + StatInt_waitpid, + StatInt_wait3, + StatInt_wait4, + StatInt_inet_ntop, + StatInt_inet_pton, + StatInt_inet_aton, + StatInt_getaddrinfo, + StatInt_getnameinfo, + StatInt_getsockname, + StatInt_gethostent, + StatInt_gethostbyname, + StatInt_gethostbyname2, + StatInt_gethostbyaddr, + StatInt_gethostent_r, + StatInt_gethostbyname_r, + StatInt_gethostbyname2_r, + StatInt_gethostbyaddr_r, + StatInt_getsockopt, + StatInt_modf, + StatInt_modff, + StatInt_modfl, + StatInt_getpeername, + StatInt_ioctl, + StatInt_sysinfo, + StatInt_readdir, + StatInt_readdir64, + StatInt_readdir_r, + StatInt_readdir64_r, + StatInt_ptrace, + StatInt_setlocale, + StatInt_getcwd, + StatInt_get_current_dir_name, + StatInt_strtoimax, + StatInt_strtoumax, + StatInt_mbstowcs, + StatInt_mbsrtowcs, + StatInt_mbsnrtowcs, + StatInt_wcstombs, + StatInt_wcsrtombs, + StatInt_wcsnrtombs, + StatInt_tcgetattr, + StatInt_realpath, + StatInt_canonicalize_file_name, + StatInt_confstr, + StatInt_sched_getaffinity, + StatInt_strerror, + StatInt_strerror_r, + StatInt_scandir, + StatInt_scandir64, + StatInt_getgroups, + StatInt_wordexp, + StatInt_sigwait, + StatInt_sigwaitinfo, + StatInt_sigtimedwait, + StatInt_sigemptyset, + StatInt_sigfillset, + StatInt_sigpending, + StatInt_sigprocmask, + StatInt_backtrace, + StatInt_backtrace_symbols, // Dynamic annotations. StatAnnotation, diff --git a/libsanitizer/tsan/tsan_suppressions.cc b/libsanitizer/tsan/tsan_suppressions.cc index 8ee8de7c278..912b3834e87 100644 --- a/libsanitizer/tsan/tsan_suppressions.cc +++ b/libsanitizer/tsan/tsan_suppressions.cc @@ -11,12 +11,25 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_suppressions.h" #include "tsan_suppressions.h" #include "tsan_rtl.h" #include "tsan_flags.h" #include "tsan_mman.h" #include "tsan_platform.h" +// Suppressions for true/false positives in standard libraries. +static const char *const std_suppressions = +// Libstdc++ 4.4 has data races in std::string. +// See http://crbug.com/181502 for an example. +"race:^_M_rep$\n" +"race:^_M_is_leaked$\n" +// False positive when using std <thread>. +// Happens because we miss atomic synchronization in libstdc++. +// See http://llvm.org/bugs/show_bug.cgi?id=17066 for details. +"race:std::_Sp_counted_ptr_inplace<std::thread::_Impl\n"; + // Can be overriden in frontend. #ifndef TSAN_GO extern "C" const char *WEAK __tsan_default_suppressions() { @@ -26,7 +39,7 @@ extern "C" const char *WEAK __tsan_default_suppressions() { namespace __tsan { -static Suppression *g_suppressions; +static SuppressionContext* g_ctx; static char *ReadFile(const char *filename) { if (filename == 0 || filename[0] == 0) @@ -36,12 +49,13 @@ static char *ReadFile(const char *filename) { internal_snprintf(tmp.data(), tmp.size(), "%s", filename); else internal_snprintf(tmp.data(), tmp.size(), "%s/%s", GetPwd(), filename); - fd_t fd = OpenFile(tmp.data(), false); - if (fd == kInvalidFd) { + uptr openrv = OpenFile(tmp.data(), false); + if (internal_iserror(openrv)) { Printf("ThreadSanitizer: failed to open suppressions file '%s'\n", tmp.data()); Die(); } + fd_t fd = openrv; const uptr fsize = internal_filesize(fd); if (fsize == (uptr)-1) { Printf("ThreadSanitizer: failed to stat suppressions file '%s'\n", @@ -59,114 +73,91 @@ static char *ReadFile(const char *filename) { return buf; } -bool SuppressionMatch(char *templ, const char *str) { - if (str == 0 || str[0] == 0) - return false; - char *tpos; - const char *spos; - while (templ && templ[0]) { - if (templ[0] == '*') { - templ++; - continue; - } - if (str[0] == 0) - return false; - tpos = (char*)internal_strchr(templ, '*'); - if (tpos != 0) - tpos[0] = 0; - spos = internal_strstr(str, templ); - str = spos + internal_strlen(templ); - templ = tpos; - if (tpos) - tpos[0] = '*'; - if (spos == 0) - return false; - } - return true; -} - -Suppression *SuppressionParse(Suppression *head, const char* supp) { - const char *line = supp; - while (line) { - while (line[0] == ' ' || line[0] == '\t') - line++; - const char *end = internal_strchr(line, '\n'); - if (end == 0) - end = line + internal_strlen(line); - if (line != end && line[0] != '#') { - const char *end2 = end; - while (line != end2 && (end2[-1] == ' ' || end2[-1] == '\t')) - end2--; - SuppressionType stype; - if (0 == internal_strncmp(line, "race:", sizeof("race:") - 1)) { - stype = SuppressionRace; - line += sizeof("race:") - 1; - } else if (0 == internal_strncmp(line, "thread:", - sizeof("thread:") - 1)) { - stype = SuppressionThread; - line += sizeof("thread:") - 1; - } else if (0 == internal_strncmp(line, "mutex:", - sizeof("mutex:") - 1)) { - stype = SuppressionMutex; - line += sizeof("mutex:") - 1; - } else if (0 == internal_strncmp(line, "signal:", - sizeof("signal:") - 1)) { - stype = SuppressionSignal; - line += sizeof("signal:") - 1; - } else { - Printf("ThreadSanitizer: failed to parse suppressions file\n"); - Die(); - } - Suppression *s = (Suppression*)internal_alloc(MBlockSuppression, - sizeof(Suppression)); - s->next = head; - head = s; - s->type = stype; - s->templ = (char*)internal_alloc(MBlockSuppression, end2 - line + 1); - internal_memcpy(s->templ, line, end2 - line); - s->templ[end2 - line] = 0; - } - if (end[0] == 0) - break; - line = end + 1; - } - return head; -} - void InitializeSuppressions() { + ALIGNED(64) static char placeholder_[sizeof(SuppressionContext)]; + g_ctx = new(placeholder_) SuppressionContext; const char *supp = ReadFile(flags()->suppressions); - g_suppressions = SuppressionParse(0, supp); + g_ctx->Parse(supp); #ifndef TSAN_GO supp = __tsan_default_suppressions(); - g_suppressions = SuppressionParse(g_suppressions, supp); + g_ctx->Parse(supp); + g_ctx->Parse(std_suppressions); #endif } -uptr IsSuppressed(ReportType typ, const ReportStack *stack) { - if (g_suppressions == 0 || stack == 0) - return 0; - SuppressionType stype; +SuppressionType conv(ReportType typ) { if (typ == ReportTypeRace) - stype = SuppressionRace; + return SuppressionRace; + else if (typ == ReportTypeVptrRace) + return SuppressionRace; + else if (typ == ReportTypeUseAfterFree) + return SuppressionRace; else if (typ == ReportTypeThreadLeak) - stype = SuppressionThread; + return SuppressionThread; else if (typ == ReportTypeMutexDestroyLocked) - stype = SuppressionMutex; + return SuppressionMutex; else if (typ == ReportTypeSignalUnsafe) - stype = SuppressionSignal; - else + return SuppressionSignal; + else if (typ == ReportTypeErrnoInSignal) + return SuppressionNone; + Printf("ThreadSanitizer: unknown report type %d\n", typ), + Die(); +} + +uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) { + CHECK(g_ctx); + if (!g_ctx->SuppressionCount() || stack == 0) return 0; + SuppressionType stype = conv(typ); + if (stype == SuppressionNone) return 0; + Suppression *s; for (const ReportStack *frame = stack; frame; frame = frame->next) { - for (Suppression *supp = g_suppressions; supp; supp = supp->next) { - if (stype == supp->type && - (SuppressionMatch(supp->templ, frame->func) || - SuppressionMatch(supp->templ, frame->file) || - SuppressionMatch(supp->templ, frame->module))) { - DPrintf("ThreadSanitizer: matched suppression '%s'\n", supp->templ); - return frame->pc; - } + if (g_ctx->Match(frame->func, stype, &s) || + g_ctx->Match(frame->file, stype, &s) || + g_ctx->Match(frame->module, stype, &s)) { + DPrintf("ThreadSanitizer: matched suppression '%s'\n", s->templ); + s->hit_count++; + *sp = s; + return frame->pc; } } return 0; } + +uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) { + CHECK(g_ctx); + if (!g_ctx->SuppressionCount() || loc == 0 || + loc->type != ReportLocationGlobal) + return 0; + SuppressionType stype = conv(typ); + if (stype == SuppressionNone) + return 0; + Suppression *s; + if (g_ctx->Match(loc->name, stype, &s) || + g_ctx->Match(loc->file, stype, &s) || + g_ctx->Match(loc->module, stype, &s)) { + DPrintf("ThreadSanitizer: matched suppression '%s'\n", s->templ); + s->hit_count++; + *sp = s; + return loc->addr; + } + return 0; +} + +void PrintMatchedSuppressions() { + CHECK(g_ctx); + InternalMmapVector<Suppression *> matched(1); + g_ctx->GetMatched(&matched); + if (!matched.size()) + return; + int hit_count = 0; + for (uptr i = 0; i < matched.size(); i++) + hit_count += matched[i]->hit_count; + Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count, + (int)internal_getpid()); + for (uptr i = 0; i < matched.size(); i++) { + Printf("%d %s:%s\n", matched[i]->hit_count, + SuppressionTypeString(matched[i]->type), matched[i]->templ); + } +} } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_suppressions.h b/libsanitizer/tsan/tsan_suppressions.h index d44d8dd3529..e38d81ece85 100644 --- a/libsanitizer/tsan/tsan_suppressions.h +++ b/libsanitizer/tsan/tsan_suppressions.h @@ -11,30 +11,15 @@ #ifndef TSAN_SUPPRESSIONS_H #define TSAN_SUPPRESSIONS_H +#include "sanitizer_common/sanitizer_suppressions.h" #include "tsan_report.h" namespace __tsan { void InitializeSuppressions(); -void FinalizeSuppressions(); -uptr IsSuppressed(ReportType typ, const ReportStack *stack); - -// Exposed for testing. -enum SuppressionType { - SuppressionRace, - SuppressionMutex, - SuppressionThread, - SuppressionSignal -}; - -struct Suppression { - Suppression *next; - SuppressionType type; - char *templ; -}; - -Suppression *SuppressionParse(Suppression *head, const char* supp); -bool SuppressionMatch(char *templ, const char *str); +void PrintMatchedSuppressions(); +uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp); +uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp); } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_symbolize.cc b/libsanitizer/tsan/tsan_symbolize.cc index 65a994670b5..0c7efac7a26 100644 --- a/libsanitizer/tsan/tsan_symbolize.cc +++ b/libsanitizer/tsan/tsan_symbolize.cc @@ -67,16 +67,65 @@ static ReportStack *NewReportStackEntry(const AddressInfo &info) { return ent; } + + ReportStack *next; + char *module; + uptr offset; + uptr pc; + char *func; + char *file; + int line; + int col; + + +// Denotes fake PC values that come from JIT/JAVA/etc. +// For such PC values __tsan_symbolize_external() will be called. +const uptr kExternalPCBit = 1ULL << 60; + +// May be overriden by JIT/JAVA/etc, +// whatever produces PCs marked with kExternalPCBit. +extern "C" bool __tsan_symbolize_external(uptr pc, + char *func_buf, uptr func_siz, + char *file_buf, uptr file_siz, + int *line, int *col) + SANITIZER_WEAK_ATTRIBUTE; + +bool __tsan_symbolize_external(uptr pc, + char *func_buf, uptr func_siz, + char *file_buf, uptr file_siz, + int *line, int *col) { + return false; +} + ReportStack *SymbolizeCode(uptr addr) { - if (!IsSymbolizerAvailable()) + // Check if PC comes from non-native land. + if (addr & kExternalPCBit) { + // Declare static to not consume too much stack space. + // We symbolize reports in a single thread, so this is fine. + static char func_buf[1024]; + static char file_buf[1024]; + int line, col; + if (!__tsan_symbolize_external(addr, func_buf, sizeof(func_buf), + file_buf, sizeof(file_buf), &line, &col)) + return NewReportStackEntry(addr); + ReportStack *ent = NewReportStackEntry(addr); + ent->module = 0; + ent->offset = 0; + ent->func = internal_strdup(func_buf); + ent->file = internal_strdup(file_buf); + ent->line = line; + ent->col = col; + return ent; + } + if (!getSymbolizer()->IsAvailable()) return SymbolizeCodeAddr2Line(addr); ScopedInSymbolizer in_symbolizer; static const uptr kMaxAddrFrames = 16; InternalScopedBuffer<AddressInfo> addr_frames(kMaxAddrFrames); for (uptr i = 0; i < kMaxAddrFrames; i++) new(&addr_frames[i]) AddressInfo(); - uptr addr_frames_num = __sanitizer::SymbolizeCode(addr, addr_frames.data(), - kMaxAddrFrames); + uptr addr_frames_num = + getSymbolizer()->SymbolizeCode(addr, addr_frames.data(), kMaxAddrFrames); if (addr_frames_num == 0) return NewReportStackEntry(addr); ReportStack *top = 0; @@ -95,11 +144,11 @@ ReportStack *SymbolizeCode(uptr addr) { } ReportLocation *SymbolizeData(uptr addr) { - if (!IsSymbolizerAvailable()) + if (!getSymbolizer()->IsAvailable()) return 0; ScopedInSymbolizer in_symbolizer; DataInfo info; - if (!__sanitizer::SymbolizeData(addr, &info)) + if (!getSymbolizer()->SymbolizeData(addr, &info)) return 0; ReportLocation *ent = (ReportLocation*)internal_alloc(MBlockReportStack, sizeof(ReportLocation)); @@ -114,4 +163,11 @@ ReportLocation *SymbolizeData(uptr addr) { return ent; } +void SymbolizeFlush() { + if (!getSymbolizer()->IsAvailable()) + return; + ScopedInSymbolizer in_symbolizer; + getSymbolizer()->Flush(); +} + } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_symbolize.h b/libsanitizer/tsan/tsan_symbolize.h index 5275936a293..c610d1f5492 100644 --- a/libsanitizer/tsan/tsan_symbolize.h +++ b/libsanitizer/tsan/tsan_symbolize.h @@ -18,6 +18,7 @@ namespace __tsan { ReportStack *SymbolizeCode(uptr addr); ReportLocation *SymbolizeData(uptr addr); +void SymbolizeFlush(); ReportStack *SymbolizeCodeAddr2Line(uptr addr); diff --git a/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc b/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc index 9bdd1ffdc5e..da0c87d297e 100644 --- a/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc +++ b/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc @@ -85,7 +85,8 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { DlIteratePhdrCtx *ctx = (DlIteratePhdrCtx*)arg; InternalScopedBuffer<char> tmp(128); if (ctx->is_first) { - internal_snprintf(tmp.data(), tmp.size(), "/proc/%d/exe", GetPid()); + internal_snprintf(tmp.data(), tmp.size(), "/proc/%d/exe", + (int)internal_getpid()); info->dlpi_name = tmp.data(); } ctx->is_first = false; diff --git a/libsanitizer/tsan/tsan_sync.cc b/libsanitizer/tsan/tsan_sync.cc index d392408fd88..04fef615531 100644 --- a/libsanitizer/tsan/tsan_sync.cc +++ b/libsanitizer/tsan/tsan_sync.cc @@ -61,7 +61,7 @@ SyncVar* SyncTab::Create(ThreadState *thr, uptr pc, uptr addr) { const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed); SyncVar *res = new(mem) SyncVar(addr, uid); #ifndef TSAN_GO - res->creation_stack.ObtainCurrent(thr, pc); + res->creation_stack_id = CurrentStackId(thr, pc); #endif return res; } @@ -80,9 +80,10 @@ SyncVar* SyncTab::GetAndLock(ThreadState *thr, uptr pc, // the hashmap anyway. if (PrimaryAllocator::PointerIsMine((void*)addr)) { MBlock *b = user_mblock(thr, (void*)addr); - Lock l(&b->mtx); + CHECK_NE(b, 0); + MBlock::ScopedLock l(b); SyncVar *res = 0; - for (res = b->head; res; res = res->next) { + for (res = b->ListHead(); res; res = res->next) { if (res->addr == addr) break; } @@ -90,8 +91,7 @@ SyncVar* SyncTab::GetAndLock(ThreadState *thr, uptr pc, if (!create) return 0; res = Create(thr, pc, addr); - res->next = b->head; - b->head = res; + b->ListPush(res); } if (write_lock) res->mtx.Lock(); @@ -145,27 +145,37 @@ SyncVar* SyncTab::GetAndRemove(ThreadState *thr, uptr pc, uptr addr) { } if (PrimaryAllocator::PointerIsMine((void*)addr)) { MBlock *b = user_mblock(thr, (void*)addr); + CHECK_NE(b, 0); SyncVar *res = 0; { - Lock l(&b->mtx); - SyncVar **prev = &b->head; - res = *prev; - while (res) { + MBlock::ScopedLock l(b); + res = b->ListHead(); + if (res) { if (res->addr == addr) { if (res->is_linker_init) return 0; - *prev = res->next; - break; + b->ListPop(); + } else { + SyncVar **prev = &res->next; + res = *prev; + while (res) { + if (res->addr == addr) { + if (res->is_linker_init) + return 0; + *prev = res->next; + break; + } + prev = &res->next; + res = *prev; + } + } + if (res) { + StatInc(thr, StatSyncDestroyed); + res->mtx.Lock(); + res->mtx.Unlock(); } - prev = &res->next; - res = *prev; } } - if (res) { - StatInc(thr, StatSyncDestroyed); - res->mtx.Lock(); - res->mtx.Unlock(); - } return res; } #endif @@ -195,26 +205,6 @@ SyncVar* SyncTab::GetAndRemove(ThreadState *thr, uptr pc, uptr addr) { return res; } -uptr SyncVar::GetMemoryConsumption() { - return sizeof(*this) - + clock.size() * sizeof(u64) - + read_clock.size() * sizeof(u64) - + creation_stack.Size() * sizeof(uptr); -} - -uptr SyncTab::GetMemoryConsumption(uptr *nsync) { - uptr mem = 0; - for (int i = 0; i < kPartCount; i++) { - Part *p = &tab_[i]; - Lock l(&p->mtx); - for (SyncVar *s = p->val; s; s = s->next) { - *nsync += 1; - mem += s->GetMemoryConsumption(); - } - } - return mem; -} - int SyncTab::PartIdx(uptr addr) { return (addr >> 3) % kPartCount; } diff --git a/libsanitizer/tsan/tsan_sync.h b/libsanitizer/tsan/tsan_sync.h index 4dbb055a17e..2867a8ac79e 100644 --- a/libsanitizer/tsan/tsan_sync.h +++ b/libsanitizer/tsan/tsan_sync.h @@ -57,7 +57,7 @@ struct SyncVar { const u64 uid; // Globally unique id. SyncClock clock; SyncClock read_clock; // Used for rw mutexes only. - StackTrace creation_stack; + u32 creation_stack_id; int owner_tid; // Set only by exclusive owners. u64 last_lock; int recursion; diff --git a/libsanitizer/tsan/tsan_update_shadow_word_inl.h b/libsanitizer/tsan/tsan_update_shadow_word_inl.h index b9aa51c7957..42caf80d349 100644 --- a/libsanitizer/tsan/tsan_update_shadow_word_inl.h +++ b/libsanitizer/tsan/tsan_update_shadow_word_inl.h @@ -55,8 +55,7 @@ do { goto RACE; } // Do the memory access intersect? - // In Go all memory accesses are 1 byte, so there can be no intersections. - if (kCppMode && Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { + if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { StatInc(thr, StatShadowIntersect); if (Shadow::TidsAreEqual(old, cur)) { StatInc(thr, StatShadowSameThread); diff --git a/libsanitizer/tsan/tsan_vector.h b/libsanitizer/tsan/tsan_vector.h index 99e9f792c20..4da8b83d5c3 100644 --- a/libsanitizer/tsan/tsan_vector.h +++ b/libsanitizer/tsan/tsan_vector.h @@ -62,6 +62,11 @@ class Vector { return &end_[-1]; } + void PopBack() { + DCHECK_GT(end_, begin_); + end_--; + } + void Resize(uptr size) { uptr old_size = Size(); EnsureSize(size); |