diff options
Diffstat (limited to 'lib/tsan')
94 files changed, 867 insertions, 529 deletions
diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt index e1da319cc..43dbe8644 100644 --- a/lib/tsan/CMakeLists.txt +++ b/lib/tsan/CMakeLists.txt @@ -61,7 +61,6 @@ set(TSAN_CXX_SOURCES if(APPLE) list(APPEND TSAN_SOURCES rtl/tsan_interceptors_mac.cc - rtl/tsan_libdispatch_mac.cc rtl/tsan_platform_mac.cc rtl/tsan_platform_posix.cc) elseif(UNIX) @@ -71,6 +70,11 @@ elseif(UNIX) rtl/tsan_platform_posix.cc) endif() +if(COMPILER_RT_INTERCEPT_LIBDISPATCH) + list(APPEND TSAN_SOURCES rtl/tsan_libdispatch.cc) + list(APPEND TSAN_RTL_CFLAGS ${COMPILER_RT_LIBDISPATCH_CFLAGS}) +endif() + set(TSAN_HEADERS rtl/tsan_clock.h rtl/tsan_defs.h @@ -236,6 +240,7 @@ endif() # Build libcxx instrumented with TSan. if(COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH AND COMPILER_RT_TEST_COMPILER_ID STREQUAL "Clang" AND NOT ANDROID) set(libcxx_tsan_deps) diff --git a/lib/tsan/benchmarks/func_entry_exit.cc b/lib/tsan/benchmarks/func_entry_exit.cc new file mode 100644 index 000000000..5e0ba1d69 --- /dev/null +++ b/lib/tsan/benchmarks/func_entry_exit.cc @@ -0,0 +1,20 @@ +// Synthetic benchmark for __tsan_func_entry/exit (spends ~75% there). + +void foo(bool x); + +int main() { + volatile int kRepeat1 = 1 << 30; + const int kRepeat = kRepeat1; + for (int i = 0; i < kRepeat; i++) + foo(false); +} + +__attribute__((noinline)) void bar(volatile bool x) { + if (x) + foo(x); +} + +__attribute__((noinline)) void foo(bool x) { + if (__builtin_expect(x, false)) + bar(x); +} diff --git a/lib/tsan/benchmarks/mop.cc b/lib/tsan/benchmarks/mop.cc new file mode 100644 index 000000000..e87fab856 --- /dev/null +++ b/lib/tsan/benchmarks/mop.cc @@ -0,0 +1,80 @@ +// Synthetic benchmark for __tsan_read/write{1,2,4,8}. +// As compared to mini_bench_local/shared.cc this benchmark passes through +// deduplication logic (ContainsSameAccess). +// First argument is access size (1, 2, 4, 8). Second optional arg switches +// from writes to reads. + +#include <pthread.h> +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/syscall.h> +#include <sys/time.h> + +template<typename T, bool write> +void* thread(void *arg) { + const int kSize = 2 << 10; + static volatile long data[kSize]; + static volatile long turn; + const int kRepeat = 1 << 17; + const int id = !!arg; + for (int i = 0; i < kRepeat; i++) { + for (;;) { + int t = __atomic_load_n(&turn, __ATOMIC_ACQUIRE); + if (t == id) + break; + syscall(SYS_futex, &turn, FUTEX_WAIT, t, 0, 0, 0); + } + for (int j = 0; j < kSize; j++) { + if (write) { + ((volatile T*)&data[j])[0] = 1; + ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1] = 1; + } else { + T v0 = ((volatile T*)&data[j])[0]; + T v1 = ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1]; + (void)v0; + (void)v1; + } + } + __atomic_store_n(&turn, 1 - id, __ATOMIC_RELEASE); + syscall(SYS_futex, &turn, FUTEX_WAKE, 0, 0, 0, 0); + } + return 0; +} + +template<typename T, bool write> +void test() { + pthread_t th; + pthread_create(&th, 0, thread<T, write>, (void*)1); + thread<T, write>(0); + pthread_join(th, 0); +} + +template<bool write> +void testw(int size) { + switch (size) { + case 1: return test<char, write>(); + case 2: return test<short, write>(); + case 4: return test<int, write>(); + case 8: return test<long long, write>(); + } +} + +int main(int argc, char** argv) { + int size = 8; + bool write = true; + if (argc > 1) { + size = atoi(argv[1]); + if (size != 1 && size != 2 && size != 4 && size != 8) + size = 8; + } + if (argc > 2) + write = false; + printf("%s%d\n", write ? "write" : "read", size); + if (write) + testw<true>(size); + else + testw<false>(size); + return 0; +} diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh index 65c34d466..a2a7e82b4 100755 --- a/lib/tsan/check_analyze.sh +++ b/lib/tsan/check_analyze.sh @@ -34,16 +34,16 @@ check() { fi } -for f in write1 write2 write4 write8 read2 read4; do +for f in write1 write2 write4 write8; do check $f rsp 1 - check $f push 1 - check $f pop 6 + check $f push 2 + check $f pop 16 done -for f in read1 read8; do +for f in read1 read2 read4 read8; do check $f rsp 1 - check $f push 2 - check $f pop 12 + check $f push 3 + check $f pop 24 done for f in func_entry func_exit; do diff --git a/lib/tsan/dd/CMakeLists.txt b/lib/tsan/dd/CMakeLists.txt index f2b8a6d17..c3f5915dd 100644 --- a/lib/tsan/dd/CMakeLists.txt +++ b/lib/tsan/dd/CMakeLists.txt @@ -10,7 +10,7 @@ set(DD_SOURCES dd_interceptors.cc ) -set(DD_LINKLIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(DD_LINKLIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl DD_LINKLIBS) append_list_if(COMPILER_RT_HAS_LIBRT rt DD_LINKLIBS) diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc index a39218f04..35a72eb36 100644 --- a/lib/tsan/dd/dd_interceptors.cc +++ b/lib/tsan/dd/dd_interceptors.cc @@ -1,9 +1,8 @@ //===-- dd_interceptors.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/tsan/dd/dd_rtl.cc b/lib/tsan/dd/dd_rtl.cc index 99b8ee597..3a9aa215d 100644 --- a/lib/tsan/dd/dd_rtl.cc +++ b/lib/tsan/dd/dd_rtl.cc @@ -1,9 +1,8 @@ //===-- dd_rtl.cc ---------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/tsan/dd/dd_rtl.h b/lib/tsan/dd/dd_rtl.h index 9abf17da4..ffe068430 100644 --- a/lib/tsan/dd/dd_rtl.h +++ b/lib/tsan/dd/dd_rtl.h @@ -1,9 +1,8 @@ //===-- dd_rtl.h ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef DD_RTL_H diff --git a/lib/tsan/go/test.c b/lib/tsan/go/test.c index c484774ca..61be48442 100644 --- a/lib/tsan/go/test.c +++ b/lib/tsan/go/test.c @@ -1,9 +1,8 @@ //===-- test.c ------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/go/tsan_go.cc b/lib/tsan/go/tsan_go.cc index 5f2507b7d..dfd1e1da1 100644 --- a/lib/tsan/go/tsan_go.cc +++ b/lib/tsan/go/tsan_go.cc @@ -1,9 +1,8 @@ //===-- tsan_go.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -214,7 +213,7 @@ void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) { ThreadState *thr = AllocGoroutine(); *pthr = thr; int goid = ThreadCreate(parent, (uptr)pc, 0, true); - ThreadStart(thr, goid, 0, /*workerthread*/ false); + ThreadStart(thr, goid, 0, ThreadType::Regular); } void __tsan_go_end(ThreadState *thr) { diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index ef984a45c..685ca5518 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -1,9 +1,8 @@ //===-- tsan_clock.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h index a891d7bbd..6a1d15a2a 100644 --- a/lib/tsan/rtl/tsan_clock.h +++ b/lib/tsan/rtl/tsan_clock.h @@ -1,9 +1,8 @@ //===-- tsan_clock.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_debugging.cc b/lib/tsan/rtl/tsan_debugging.cc index 067aeef97..8579db12b 100644 --- a/lib/tsan/rtl/tsan_debugging.cc +++ b/lib/tsan/rtl/tsan_debugging.cc @@ -1,9 +1,8 @@ //===-- tsan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h index 3c775debf..293d7decc 100644 --- a/lib/tsan/rtl/tsan_defs.h +++ b/lib/tsan/rtl/tsan_defs.h @@ -1,9 +1,8 @@ //===-- tsan_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index 16dbdf391..64fc50e95 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dispatch_defs.h b/lib/tsan/rtl/tsan_dispatch_defs.h new file mode 100644 index 000000000..6f1d1f75f --- /dev/null +++ b/lib/tsan/rtl/tsan_dispatch_defs.h @@ -0,0 +1,66 @@ +//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_DISPATCH_DEFS_H +#define TSAN_DISPATCH_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +typedef struct dispatch_object_s {} *dispatch_object_t; + +#define DISPATCH_DECL(name) \ + typedef struct name##_s : public dispatch_object_s {} *name##_t + +DISPATCH_DECL(dispatch_queue); +DISPATCH_DECL(dispatch_source); +DISPATCH_DECL(dispatch_group); +DISPATCH_DECL(dispatch_data); +DISPATCH_DECL(dispatch_semaphore); +DISPATCH_DECL(dispatch_io); + +typedef void (*dispatch_function_t)(void *arg); +typedef void (^dispatch_block_t)(void); +typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, + int error); + +typedef long dispatch_once_t; // NOLINT +typedef __sanitizer::u64 dispatch_time_t; +typedef int dispatch_fd_t; // NOLINT +typedef unsigned long dispatch_io_type_t; // NOLINT +typedef unsigned long dispatch_io_close_flags_t; // NOLINT + +extern "C" { +void *dispatch_get_context(dispatch_object_t object); +void dispatch_retain(dispatch_object_t object); +void dispatch_release(dispatch_object_t object); + +extern const dispatch_block_t _dispatch_data_destructor_free; +extern const dispatch_block_t _dispatch_data_destructor_munmap; +} // extern "C" + +#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr +#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free +#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap + +#if __has_attribute(noescape) + #define DISPATCH_NOESCAPE __attribute__((__noescape__)) +#else + #define DISPATCH_NOESCAPE +#endif + +// Data types used in dispatch APIs +typedef unsigned long size_t; // NOLINT +typedef unsigned long uintptr_t; // NOLINT +typedef __sanitizer::s64 off_t; +typedef __sanitizer::u16 mode_t; +typedef long long_t; // NOLINT + +#endif // TSAN_DISPATCH_DEFS_H diff --git a/lib/tsan/rtl/tsan_external.cc b/lib/tsan/rtl/tsan_external.cc index 6c0e9477e..ba8bb71be 100644 --- a/lib/tsan/rtl/tsan_external.cc +++ b/lib/tsan/rtl/tsan_external.cc @@ -1,9 +1,8 @@ //===-- tsan_external.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.cc b/lib/tsan/rtl/tsan_fd.cc index f13a7432e..5b562ae68 100644 --- a/lib/tsan/rtl/tsan_fd.cc +++ b/lib/tsan/rtl/tsan_fd.cc @@ -1,9 +1,8 @@ //===-- tsan_fd.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.h b/lib/tsan/rtl/tsan_fd.h index 64dc84f6e..ce4f2f73b 100644 --- a/lib/tsan/rtl/tsan_fd.h +++ b/lib/tsan/rtl/tsan_fd.h @@ -1,9 +1,8 @@ //===-- tsan_fd.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc index 877fc8b81..17cd919b8 100644 --- a/lib/tsan/rtl/tsan_flags.cc +++ b/lib/tsan/rtl/tsan_flags.cc @@ -1,9 +1,8 @@ //===-- tsan_flags.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.h b/lib/tsan/rtl/tsan_flags.h index 66740def5..dbf9345fa 100644 --- a/lib/tsan/rtl/tsan_flags.h +++ b/lib/tsan/rtl/tsan_flags.h @@ -1,9 +1,8 @@ //===-- tsan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index d3b678fdd..bfb74b696 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -1,9 +1,8 @@ //===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.cc b/lib/tsan/rtl/tsan_ignoreset.cc index cdb90d229..b2f657939 100644 --- a/lib/tsan/rtl/tsan_ignoreset.cc +++ b/lib/tsan/rtl/tsan_ignoreset.cc @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.h b/lib/tsan/rtl/tsan_ignoreset.h index e747d819c..3e318bd67 100644 --- a/lib/tsan/rtl/tsan_ignoreset.h +++ b/lib/tsan/rtl/tsan_ignoreset.h @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index 9e49dfe5d..e8908ac98 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -154,7 +153,7 @@ const int SIG_SETMASK = 2; #endif #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ - (!cur_thread()->is_inited) + (cur_thread_init(), !cur_thread()->is_inited) namespace __tsan { struct SignalDesc { @@ -399,7 +398,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -409,7 +408,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -455,7 +454,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -555,6 +554,7 @@ static void LongJmp(ThreadState *thr, uptr *env) { // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { + cur_thread_init(); SetJmp(cur_thread(), sp, mangled_sp); } @@ -665,7 +665,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(size); void *p = 0; { @@ -682,7 +682,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalCalloc(size, n); void *p = 0; { @@ -694,7 +694,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -706,10 +706,23 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { return p; } +TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { + if (in_symbolizer()) + return InternalReallocArray(p, size, n); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + p = user_reallocarray(thr, pc, p, size, n); + } + invoke_malloc_hook(p, size); + return p; +} + TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -719,7 +732,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -808,14 +821,14 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_valloc(thr, pc, sz); @@ -824,7 +837,7 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { uptr PageSize = GetPageSizeCached(); sz = sz ? RoundUpTo(sz, PageSize) : PageSize; return InternalAlloc(sz, nullptr, PageSize); @@ -839,7 +852,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); if (!p) return errno_ENOMEM; @@ -943,6 +956,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { void *param = p->param; int tid = 0; { + cur_thread_init(); ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; @@ -959,7 +973,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); @@ -1051,6 +1065,16 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + { + SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif + } + REAL(pthread_exit)(retval); +} + #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); @@ -1975,6 +1999,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { + cur_thread_init(); ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -2091,7 +2116,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2204,23 +2229,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO -// There interceptors do not seem to be strictly necessary for tsan. -// But we see cases where the interceptors consume 70% of execution time. -// Memory blocks passed to fgetgrent_r are "written to" by tsan several times. -// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each -// function "writes to" the buffer. Then, the same memory is "written to" -// twice, first as buf and then as pwbufp (both of them refer to the same -// addresses). -#undef SANITIZER_INTERCEPT_GETPWENT -#undef SANITIZER_INTERCEPT_GETPWENT_R -#undef SANITIZER_INTERCEPT_FGETPWENT -#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS -#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ @@ -2620,6 +2634,9 @@ static void unreachable() { } #endif +// Define default implementation since interception of libdispatch is optional. +SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} + void InitializeInterceptors() { #if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. @@ -2637,18 +2654,18 @@ void InitializeInterceptors() { InitializeCommonInterceptors(); InitializeSignalInterceptors(); + InitializeLibdispatchInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. - using __interception::GetRealFunctionAddress; - GetRealFunctionAddress(TSAN_STRING_SETJMP, - (uptr*)&REAL(setjmp_symname), 0, 0); - GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, - (uptr*)&REAL(sigsetjmp_symname), 0, 0); + using __interception::InterceptFunction; + InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, + 0); #if !SANITIZER_NETBSD - GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif #endif @@ -2662,6 +2679,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); + TSAN_INTERCEPT(reallocarray); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(munmap); @@ -2677,6 +2695,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index 763b46b88..88d1edd77 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -21,9 +21,17 @@ class ScopedInterceptor { LibIgnore *libignore(); +#if !SANITIZER_GO +INLINE bool in_symbolizer() { + cur_thread_init(); + return UNLIKELY(cur_thread()->in_symbolizer); +} +#endif + } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc index 579c4d0c0..99c6df9df 100644 --- a/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors_mac.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,8 +20,10 @@ #include "tsan_interface_ann.h" #include "sanitizer_common/sanitizer_addrhashmap.h" +#include <errno.h> #include <libkern/OSAtomic.h> #include <objc/objc-sync.h> +#include <sys/ucontext.h> #if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> @@ -30,6 +31,11 @@ typedef long long_t; // NOLINT +extern "C" { +int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); +int setcontext(const ucontext_t *ucp); +} + namespace __tsan { // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, @@ -354,6 +360,31 @@ TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { return result; } +TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { + { + SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); + } + // Bacause of swapcontext() semantics we have no option but to copy its + // impementation here + if (!oucp || !ucp) { + errno = EINVAL; + return -1; + } + ThreadState *thr = cur_thread(); + const int UCF_SWAPPED = 0x80000000; + oucp->uc_onstack &= ~UCF_SWAPPED; + thr->ignore_interceptors++; + int ret = getcontext(oucp); + if (!(oucp->uc_onstack & UCF_SWAPPED)) { + thr->ignore_interceptors--; + if (!ret) { + oucp->uc_onstack |= UCF_SWAPPED; + ret = setcontext(ucp); + } + } + return ret; +} + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/lib/tsan/rtl/tsan_interface.cc b/lib/tsan/rtl/tsan_interface.cc index ad9b1fe9a..508aadb08 100644 --- a/lib/tsan/rtl/tsan_interface.cc +++ b/lib/tsan/rtl/tsan_interface.cc @@ -1,9 +1,8 @@ //===-- tsan_interface.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef u32 uint32_t; typedef u64 uint64_t; void __tsan_init() { + cur_thread_init(); Initialize(cur_thread()); } @@ -124,6 +124,31 @@ void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { __tsan_unaligned_write8(addr); *addr = v; } + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast<ThreadState *>(fiber), name); +} } // extern "C" void __tsan_acquire(void *addr) { diff --git a/lib/tsan/rtl/tsan_interface.h b/lib/tsan/rtl/tsan_interface.h index 203f6b106..fac57809a 100644 --- a/lib/tsan/rtl/tsan_interface.h +++ b/lib/tsan/rtl/tsan_interface.h @@ -1,9 +1,8 @@ //===-- tsan_interface.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -200,7 +199,7 @@ __extension__ typedef __int128 a128; #endif // Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic typedef enum { mo_relaxed, mo_consume, diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index a7922a42e..e141ddbb7 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_ann.h b/lib/tsan/rtl/tsan_interface_ann.h index 963bcc55a..458d61f53 100644 --- a/lib/tsan/rtl/tsan_interface_ann.h +++ b/lib/tsan/rtl/tsan_interface_ann.h @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index d334394f5..a6b7b0f65 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_atomic.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -475,7 +474,7 @@ static morder convert_morder(morder mo) { #define SCOPED_ATOMIC(func, ...) \ ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ diff --git a/lib/tsan/rtl/tsan_interface_inl.h b/lib/tsan/rtl/tsan_interface_inl.h index fff83ee17..bf4a16586 100644 --- a/lib/tsan/rtl/tsan_interface_inl.h +++ b/lib/tsan/rtl/tsan_interface_inl.h @@ -1,9 +1,8 @@ //===-- tsan_interface_inl.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.cc b/lib/tsan/rtl/tsan_interface_java.cc index 75e960e62..9f227f095 100644 --- a/lib/tsan/rtl/tsan_interface_java.cc +++ b/lib/tsan/rtl/tsan_interface_java.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_java.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.h b/lib/tsan/rtl/tsan_interface_java.h index 0bd49ac3c..5ad16e959 100644 --- a/lib/tsan/rtl/tsan_interface_java.h +++ b/lib/tsan/rtl/tsan_interface_java.h @@ -1,9 +1,8 @@ //===-- tsan_interface_java.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch.cc index df22888b3..48ac6a282 100644 --- a/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/lib/tsan/rtl/tsan_libdispatch.cc @@ -1,38 +1,26 @@ -//===-- tsan_libdispatch_mac.cc -------------------------------------------===// +//===-- tsan_libdispatch.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Mac-specific libdispatch (GCD) support. +// Support for intercepting libdispatch (GCD). //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC - #include "sanitizer_common/sanitizer_common.h" #include "interception/interception.h" #include "tsan_interceptors.h" -#include "tsan_platform.h" #include "tsan_rtl.h" -#include <Block.h> -#include <dispatch/dispatch.h> -#include <pthread.h> - -// DISPATCH_NOESCAPE is not defined prior to XCode 8. -#ifndef DISPATCH_NOESCAPE -#define DISPATCH_NOESCAPE -#endif - -typedef long long_t; // NOLINT +#include "BlocksRuntime/Block.h" +#include "tsan_dispatch_defs.h" namespace __tsan { + typedef u16 uint16_t; typedef struct { dispatch_queue_t queue; @@ -42,7 +30,7 @@ typedef struct { bool submitted_synchronously; bool is_barrier_block; uptr non_queue_sync_object; -} tsan_block_context_t; +} block_context_t; // The offsets of different fields of the dispatch_queue_t structure, exported // by libdispatch.dylib. @@ -86,13 +74,11 @@ static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { return tq; } -static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, - dispatch_queue_t queue, - void *orig_context, - dispatch_function_t orig_work) { - tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc_internal(thr, pc, - sizeof(tsan_block_context_t)); +static block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, void *orig_context, + dispatch_function_t orig_work) { + block_context_t *new_context = + (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -111,7 +97,7 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, bool serial_task = context->is_barrier_block || is_queue_serial static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; Acquire(thr, pc, submit_sync); @@ -126,7 +112,7 @@ static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, } static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; if (context->submitted_synchronously) Release(thr, pc, submit_sync); @@ -142,7 +128,7 @@ static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, static void dispatch_callback_wrap(void *param) { SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); - tsan_block_context_t *context = (tsan_block_context_t *)param; + block_context_t *context = (block_context_t *)param; dispatch_sync_pre_execute(thr, pc, context); @@ -166,13 +152,13 @@ static void invoke_and_release_block(void *param) { Block_release(block); } -#define DISPATCH_INTERCEPT_B(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -185,7 +171,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -194,11 +180,11 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } -#define DISPATCH_INTERCEPT_F(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, context, work); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -211,7 +197,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, context, work, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -220,18 +206,21 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } +#define DISPATCH_INTERCEPT(name, barrier) \ + DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ + DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ + DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ + DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) + // We wrap dispatch_async, dispatch_sync and friends where we allocate a new // context, which is used to synchronize (we release the context before // submitting, and the callback acquires it before executing the original // callback). -DISPATCH_INTERCEPT_B(dispatch_async, false) -DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) -DISPATCH_INTERCEPT_F(dispatch_async_f, false) -DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) -DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) -DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) -DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) -DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) +DISPATCH_INTERCEPT(dispatch, false) +DISPATCH_INTERCEPT(dispatch_barrier, true) + +DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, dispatch_function_t work) TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, dispatch_queue_t queue, dispatch_block_t block) { @@ -239,7 +228,7 @@ TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); Release(thr, pc, (uptr)new_context); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); @@ -324,9 +313,12 @@ TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, return result; } +// Used, but not intercepted. +extern "C" void dispatch_group_enter(dispatch_group_t group); + TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); - // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. + // Acquired in the group notification callback in dispatch_group_notify[_f]. Release(thr, pc, (uptr)group); REAL(dispatch_group_leave)(group); } @@ -336,10 +328,10 @@ TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); dispatch_retain(group); dispatch_group_enter(group); - __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); + __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); WRAP(dispatch_async)(queue, ^(void) { block_copy(); - _Block_release(block_copy); + Block_release(block_copy); WRAP(dispatch_group_leave)(group); dispatch_release(group); }); @@ -358,6 +350,9 @@ TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, }); } +DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) + TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, dispatch_queue_t q, dispatch_block_t block) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); @@ -377,7 +372,7 @@ TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, block(); }); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); new_context->is_barrier_block = true; Release(thr, pc, (uptr)new_context); @@ -395,7 +390,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, if (handler == nullptr) return REAL(dispatch_source_set_event_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0 }; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -424,7 +419,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, if (handler == nullptr) return REAL(dispatch_source_set_cancel_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -455,7 +450,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, if (handler == nullptr) return REAL(dispatch_source_set_registration_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -484,34 +479,54 @@ TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); - void *parent_to_child_sync = nullptr; - uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; - void *child_to_parent_sync = nullptr; - uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; - Release(thr, pc, parent_to_child_sync_uptr); + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { SCOPED_INTERCEPTOR_RAW(dispatch_apply); - Acquire(thr, pc, parent_to_child_sync_uptr); + Acquire(thr, pc, parent_to_child_sync); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); block(iteration); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Release(thr, pc, child_to_parent_sync_uptr); + Release(thr, pc, child_to_parent_sync); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); REAL(dispatch_apply)(iterations, queue, new_block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Acquire(thr, pc, child_to_parent_sync_uptr); + Acquire(thr, pc, child_to_parent_sync); +} + +static void invoke_block_iteration(void *param, size_t iteration) { + auto block = (void (^)(size_t)) param; + block(iteration); } TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, dispatch_queue_t queue, void *context, void (*work)(void *, size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + + // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch + // implements dispatch_apply in terms of dispatch_apply_f. + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); work(context, iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); }; - WRAP(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); } DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) @@ -531,7 +546,7 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); uptr submit_sync = (uptr)new_context; Release(thr, pc, submit_sync); @@ -546,7 +561,7 @@ typedef void (^cleanup_handler_t)(int error); TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -563,7 +578,7 @@ TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -580,7 +595,7 @@ TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -599,7 +614,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, dispatch_data_t data, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -617,7 +632,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, dispatch_block_t barrier) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { nullptr, nullptr, &invoke_block, false, false, false, 0}; new_context.non_queue_sync_object = (uptr)channel; new_context.is_barrier_block = true; @@ -637,7 +652,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -662,7 +677,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -687,7 +702,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -722,6 +737,46 @@ TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { return REAL(dispatch_resume)(o); } -} // namespace __tsan +void InitializeLibdispatchInterceptors() { + INTERCEPT_FUNCTION(dispatch_async); + INTERCEPT_FUNCTION(dispatch_async_f); + INTERCEPT_FUNCTION(dispatch_sync); + INTERCEPT_FUNCTION(dispatch_sync_f); + INTERCEPT_FUNCTION(dispatch_barrier_async); + INTERCEPT_FUNCTION(dispatch_barrier_async_f); + INTERCEPT_FUNCTION(dispatch_barrier_sync); + INTERCEPT_FUNCTION(dispatch_barrier_sync_f); + INTERCEPT_FUNCTION(dispatch_after); + INTERCEPT_FUNCTION(dispatch_after_f); + INTERCEPT_FUNCTION(dispatch_once); + INTERCEPT_FUNCTION(dispatch_once_f); + INTERCEPT_FUNCTION(dispatch_semaphore_signal); + INTERCEPT_FUNCTION(dispatch_semaphore_wait); + INTERCEPT_FUNCTION(dispatch_group_wait); + INTERCEPT_FUNCTION(dispatch_group_leave); + INTERCEPT_FUNCTION(dispatch_group_async); + INTERCEPT_FUNCTION(dispatch_group_async_f); + INTERCEPT_FUNCTION(dispatch_group_notify); + INTERCEPT_FUNCTION(dispatch_group_notify_f); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); + INTERCEPT_FUNCTION(dispatch_apply); + INTERCEPT_FUNCTION(dispatch_apply_f); + INTERCEPT_FUNCTION(dispatch_data_create); + INTERCEPT_FUNCTION(dispatch_read); + INTERCEPT_FUNCTION(dispatch_write); + INTERCEPT_FUNCTION(dispatch_io_read); + INTERCEPT_FUNCTION(dispatch_io_write); + INTERCEPT_FUNCTION(dispatch_io_barrier); + INTERCEPT_FUNCTION(dispatch_io_create); + INTERCEPT_FUNCTION(dispatch_io_create_with_path); + INTERCEPT_FUNCTION(dispatch_io_create_with_io); + INTERCEPT_FUNCTION(dispatch_io_close); + INTERCEPT_FUNCTION(dispatch_resume); +} -#endif // SANITIZER_MAC +} // namespace __tsan diff --git a/lib/tsan/rtl/tsan_malloc_mac.cc b/lib/tsan/rtl/tsan_malloc_mac.cc index 3cc30724d..0b874aecb 100644 --- a/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/lib/tsan/rtl/tsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,19 +28,19 @@ using namespace __tsan; void *p = \ user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + if (in_symbolizer()) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) #define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + if (in_symbolizer()) return InternalRealloc(ptr, size); \ SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) #define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + if (in_symbolizer()) return InternalCalloc(count, size); \ SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ - if (cur_thread()->in_symbolizer) { \ + if (in_symbolizer()) { \ void *p = InternalAlloc(size, nullptr, alignment); \ if (!p) return errno_ENOMEM; \ *memptr = p; \ @@ -50,12 +49,12 @@ using namespace __tsan; SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \ int res = user_posix_memalign(thr, pc, memptr, alignment, size); #define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + if (in_symbolizer()) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); @@ -64,6 +63,8 @@ using namespace __tsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __tsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/tsan/rtl/tsan_md5.cc b/lib/tsan/rtl/tsan_md5.cc index 51279c10d..bfe0c17ba 100644 --- a/lib/tsan/rtl/tsan_md5.cc +++ b/lib/tsan/rtl/tsan_md5.cc @@ -1,9 +1,8 @@ //===-- tsan_md5.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,14 @@ static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) { return ptr; } +#undef F +#undef G +#undef H +#undef I +#undef STEP +#undef SET +#undef GET + void MD5_Init(MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index b160a9736..f4a95d870 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -1,9 +1,8 @@ //===-- tsan_mman.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,6 +201,16 @@ void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { return SetErrnoOnNull(p); } +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportReallocArrayOverflow(size, n, &stack); + } + return user_realloc(thr, pc, p, size * n); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h index 6042c5c5d..467aabdf2 100644 --- a/lib/tsan/rtl/tsan_mman.h +++ b/lib/tsan/rtl/tsan_mman.h @@ -1,9 +1,8 @@ //===-- tsan_mman.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); void *user_alloc(ThreadState *thr, uptr pc, uptr sz); void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n); void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, uptr sz); diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc index 22afefce3..bb7531325 100644 --- a/lib/tsan/rtl/tsan_mutex.cc +++ b/lib/tsan/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h index 22ee2f33f..80fdc6ed5 100644 --- a/lib/tsan/rtl/tsan_mutex.h +++ b/lib/tsan/rtl/tsan_mutex.h @@ -1,9 +1,8 @@ //===-- tsan_mutex.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.cc b/lib/tsan/rtl/tsan_mutexset.cc index 21587770f..02e5f9da6 100644 --- a/lib/tsan/rtl/tsan_mutexset.cc +++ b/lib/tsan/rtl/tsan_mutexset.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.h b/lib/tsan/rtl/tsan_mutexset.h index 605c21a9c..d63881f40 100644 --- a/lib/tsan/rtl/tsan_mutexset.h +++ b/lib/tsan/rtl/tsan_mutexset.h @@ -1,9 +1,8 @@ //===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc index 4f52d3d71..4cbdf703a 100644 --- a/lib/tsan/rtl/tsan_new_delete.cc +++ b/lib/tsan/rtl/tsan_new_delete.cc @@ -1,9 +1,8 @@ //===-- tsan_new_delete.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,7 +29,7 @@ DECLARE_REAL(void, free, void *ptr) // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. #define OPERATOR_NEW_BODY(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size); \ void *p = 0; \ { \ @@ -45,7 +44,7 @@ DECLARE_REAL(void, free, void *ptr) return p; #define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, (uptr)align); \ void *p = 0; \ { \ @@ -115,7 +114,7 @@ void *operator new[](__sanitizer::uptr size, std::align_val_t align, #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 8303c2418..03a36fd9e 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -1,9 +1,8 @@ //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index d2ce60709..d6a05386f 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_linux.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -402,6 +401,10 @@ ThreadState *cur_thread() { return thr; } +void set_cur_thread(ThreadState *thr) { + *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); +} + void cur_thread_finalize() { __sanitizer_sigset_t emptyset; internal_sigfillset(&emptyset); diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index 7e3a47387..9b9c8dbe1 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_mac.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,22 +73,22 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { // shadow memory is set up. static uptr main_thread_identity = 0; ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; +static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state; -ThreadState **cur_thread_location() { - ThreadState **thread_identity = (ThreadState **)pthread_self(); - return ((uptr)thread_identity == main_thread_identity) ? nullptr - : thread_identity; +static ThreadState **cur_thread_location() { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity || main_thread_identity == 0) + return &main_thread_state_loc; + return (ThreadState **)MemToShadow(thread_identity); } ThreadState *cur_thread() { - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr || main_thread_identity == 0) { - return (ThreadState *)&main_thread_state; - } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( - (uptr *)fake_tls, sizeof(ThreadState)); - return thr; + return (ThreadState *)SignalSafeGetOrAllocate( + (uptr *)cur_thread_location(), sizeof(ThreadState)); +} + +void set_cur_thread(ThreadState *thr) { + *cur_thread_location() = thr; } // TODO(kuba.brecka): This is not async-signal-safe. In particular, we call @@ -97,14 +96,13 @@ ThreadState *cur_thread() { // handler will try to access the unmapped ThreadState. void cur_thread_finalize() { ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + if (thr_state_loc == &main_thread_state_loc) { // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to // exit the main thread. Let's keep the main thread's ThreadState. return; } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - internal_munmap(*fake_tls, sizeof(ThreadState)); - *fake_tls = nullptr; + internal_munmap(*thr_state_loc, sizeof(ThreadState)); + *thr_state_loc = nullptr; } #endif @@ -213,7 +211,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ true); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { if (thread == pthread_self()) { @@ -266,11 +264,11 @@ void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // The pointer to the ThreadState object is stored in the shadow memory // of the tls. uptr tls_end = tls_addr + tls_size; - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity) { MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size); } else { - uptr thr_state_start = (uptr)thr_state_loc; + uptr thr_state_start = thread_identity; uptr thr_state_end = thr_state_start + sizeof(uptr); CHECK_GE(thr_state_start, tls_addr); CHECK_LE(thr_state_start, tls_addr + tls_size); diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc index c38dcc7f2..3bd0f1bd4 100644 --- a/lib/tsan/rtl/tsan_platform_posix.cc +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_posix.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_windows.cc b/lib/tsan/rtl/tsan_platform_windows.cc index 08aa588a4..037297559 100644 --- a/lib/tsan/rtl/tsan_platform_windows.cc +++ b/lib/tsan/rtl/tsan_platform_windows.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_windows.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_preinit.cc b/lib/tsan/rtl/tsan_preinit.cc index a96618d7d..052b35309 100644 --- a/lib/tsan/rtl/tsan_preinit.cc +++ b/lib/tsan/rtl/tsan_preinit.cc @@ -1,9 +1,8 @@ //===-- tsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc index 629c3e933..ae669024a 100644 --- a/lib/tsan/rtl/tsan_report.cc +++ b/lib/tsan/rtl/tsan_report.cc @@ -1,9 +1,8 @@ //===-- tsan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -258,7 +257,7 @@ static void PrintThread(const ReportThread *rt) { Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; const char *thread_status = rt->running ? "running" : "finished"; - if (rt->workerthread) { + if (rt->thread_type == ThreadType::Worker) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); Printf("%s", d.Default()); diff --git a/lib/tsan/rtl/tsan_report.h b/lib/tsan/rtl/tsan_report.h index cdc999c6a..b4e4d8989 100644 --- a/lib/tsan/rtl/tsan_report.h +++ b/lib/tsan/rtl/tsan_report.h @@ -1,9 +1,8 @@ //===-- tsan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" @@ -92,7 +92,7 @@ struct ReportThread { int id; tid_t os_id; bool running; - bool workerthread; + ThreadType thread_type; char *name; u32 parent_tid; ReportStack *stack; diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index f038e9682..3d23f5047 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -329,11 +328,8 @@ static void CheckShadowMapping() { #if !SANITIZER_GO static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - uptr top = 0; - uptr bottom = 0; - bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { @@ -397,7 +393,7 @@ void Initialize(ThreadState *thr) { // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -642,6 +638,7 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, // __m128i _mm_move_epi64(__m128i*); // _mm_storel_epi64(u64*, __m128i); u64 store_word = cur.raw(); + bool stored = false; // scan all the shadow values and dispatch to 4 categories: // same, replace, candidate and race (see comments below). @@ -666,16 +663,28 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, int idx = 0; #include "tsan_update_shadow_word_inl.h" idx = 1; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 2; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 3; + if (stored) { #include "tsan_update_shadow_word_inl.h" + } else { +#include "tsan_update_shadow_word_inl.h" + } #endif // we did not find any races and had already stored // the current access info, so we are done - if (LIKELY(store_word == 0)) + if (LIKELY(stored)) return; // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); @@ -815,7 +824,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif - if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -826,7 +835,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h index 3410be294..d58c1dca4 100644 --- a/lib/tsan/rtl/tsan_rtl.h +++ b/lib/tsan/rtl/tsan_rtl.h @@ -1,9 +1,8 @@ //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,19 +55,14 @@ namespace __tsan { #if !SANITIZER_GO struct MapUnmapCallback; #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) -static const uptr kAllocatorRegionSizeLog = 20; -static const uptr kAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - LocalAddressSpaceView, MapUnmapCallback>; + struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -85,10 +79,8 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; Allocator *allocator(); #endif @@ -385,6 +377,9 @@ struct ThreadState { // taken by epoch between synchs. // This way we can save one load from tls. u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. @@ -462,12 +457,22 @@ struct ThreadState { #if !SANITIZER_GO #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); +void set_cur_thread(ThreadState *thr); void cur_thread_finalize(); +INLINE void cur_thread_init() { } #else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { - return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); + return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; +} +INLINE void cur_thread_init() { + ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; +} +INLINE void set_cur_thread(ThreadState *thr) { + reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -765,7 +770,8 @@ void FuncEntry(ThreadState *thr, uptr pc); void FuncExit(ThreadState *thr); int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread); +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type); void ThreadFinish(ThreadState *thr); int ThreadTid(ThreadState *thr, uptr pc, uptr uid); void ThreadJoin(ThreadState *thr, uptr pc, int tid); @@ -868,6 +874,16 @@ uptr ALWAYS_INLINE HeapEnd() { } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S index 3d02bf22f..7c3ce13e4 100644 --- a/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -335,9 +335,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif -#if defined(__linux__) -/* We do not need executable stack. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index 34ef51c2a..b5c8cb7bf 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -389,10 +389,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif // !defined(__APPLE__) && !defined(__NetBSD__) -#if defined(__FreeBSD__) || defined(__linux__) -/* We do not need executable stack. */ -/* This note is not needed on NetBSD. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index c61d02b7a..941e70f98 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_mutex.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_proc.cc b/lib/tsan/rtl/tsan_rtl_proc.cc index efccdb590..94bbed25b 100644 --- a/lib/tsan/rtl/tsan_rtl_proc.cc +++ b/lib/tsan/rtl/tsan_rtl_proc.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_proc.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index febb6cef2..220a425a2 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_report.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,7 +201,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; - rt->workerthread = tctx->workerthread; + rt->thread_type = tctx->thread_type; rt->stack = 0; rt->stack = SymbolizeStackId(tctx->creation_stack_id); if (rt->stack) @@ -730,10 +729,12 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { ALWAYS_INLINE void PrintCurrentStackSlow(uptr pc) { #if !SANITIZER_GO + uptr bp = GET_CURRENT_FRAME(); BufferedStackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace))) BufferedStackTrace(); - ptrace->Unwind(kStackTraceMax, pc, 0, 0, 0, 0, false); + ptrace->Unwind(pc, bp, nullptr, false); + for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace_buffer[i]; ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1]; diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 766a0f5a5..fd95cfed4 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_thread.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,13 +239,15 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { return tid; } -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type) { uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -258,7 +259,7 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, workerthread, &args); + tr->StartThread(tid, os_id, thread_type, &args); tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); @@ -404,4 +405,40 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } } +#if !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast<ThreadState *>(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan diff --git a/lib/tsan/rtl/tsan_stack_trace.cc b/lib/tsan/rtl/tsan_stack_trace.cc index a0dee19e2..dbaca23c6 100644 --- a/lib/tsan/rtl/tsan_stack_trace.cc +++ b/lib/tsan/rtl/tsan_stack_trace.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,3 +48,16 @@ void VarSizeStackTrace::ReverseOrder() { } } // namespace __tsan + +#if !SANITIZER_GO +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + uptr top = 0; + uptr bottom = 0; + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} +#endif // SANITIZER_GO diff --git a/lib/tsan/rtl/tsan_stack_trace.h b/lib/tsan/rtl/tsan_stack_trace.h index f69b57464..3eb8ce156 100644 --- a/lib/tsan/rtl/tsan_stack_trace.h +++ b/lib/tsan/rtl/tsan_stack_trace.h @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 18c83d5c6..d23ff47d9 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -1,9 +1,8 @@ //===-- tsan_stat.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 42d6a2b63..94e18bc66 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -1,9 +1,8 @@ //===-- tsan_stat.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.cc b/lib/tsan/rtl/tsan_suppressions.cc index 6df074118..b3eea9ab5 100644 --- a/lib/tsan/rtl/tsan_suppressions.cc +++ b/lib/tsan/rtl/tsan_suppressions.cc @@ -1,9 +1,8 @@ //===-- tsan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.h b/lib/tsan/rtl/tsan_suppressions.h index 526952d5b..f430aeb6c 100644 --- a/lib/tsan/rtl/tsan_suppressions.h +++ b/lib/tsan/rtl/tsan_suppressions.h @@ -1,9 +1,8 @@ //===-- tsan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.cc b/lib/tsan/rtl/tsan_symbolize.cc index 27f0e01c7..cb60763f4 100644 --- a/lib/tsan/rtl/tsan_symbolize.cc +++ b/lib/tsan/rtl/tsan_symbolize.cc @@ -1,9 +1,8 @@ //===-- tsan_symbolize.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.h b/lib/tsan/rtl/tsan_symbolize.h index 5a9710a3c..7adaa04dc 100644 --- a/lib/tsan/rtl/tsan_symbolize.h +++ b/lib/tsan/rtl/tsan_symbolize.h @@ -1,9 +1,8 @@ //===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc index ba3953375..c613b116e 100644 --- a/lib/tsan/rtl/tsan_sync.cc +++ b/lib/tsan/rtl/tsan_sync.cc @@ -1,9 +1,8 @@ //===-- tsan_sync.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h index 9039970bc..47f2739d8 100644 --- a/lib/tsan/rtl/tsan_sync.h +++ b/lib/tsan/rtl/tsan_sync.h @@ -1,9 +1,8 @@ //===-- tsan_sync.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_trace.h b/lib/tsan/rtl/tsan_trace.h index 9aef375cb..fbd0f72db 100644 --- a/lib/tsan/rtl/tsan_trace.h +++ b/lib/tsan/rtl/tsan_trace.h @@ -1,9 +1,8 @@ //===-- tsan_trace.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_update_shadow_word_inl.h b/lib/tsan/rtl/tsan_update_shadow_word_inl.h index 6e3ac2fa1..056c3aa20 100644 --- a/lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ b/lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -1,9 +1,8 @@ //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,31 +17,35 @@ do { const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); - if (old.IsZero()) { + if (LIKELY(old.IsZero())) { StatInc(thr, StatShadowZero); - if (store_word) + if (!stored) { StoreIfNotYetStored(sp, &store_word); - // The above StoreIfNotYetStored could be done unconditionally - // and it even shows 4% gain on synthetic benchmarks (r4307). + stored = true; + } break; } // is the memory access equal to the previous? - if (Shadow::Addr0AndSizeAreEqual(cur, old)) { + if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { StatInc(thr, StatShadowSameSize); // same thread? - if (Shadow::TidsAreEqual(old, cur)) { + if (LIKELY(Shadow::TidsAreEqual(old, cur))) { StatInc(thr, StatShadowSameThread); - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } StatInc(thr, StatShadowAnotherThread); if (HappensBefore(old, thr)) { - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +59,7 @@ do { StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt index 352319f10..7b1ba21c5 100644 --- a/lib/tsan/tests/CMakeLists.txt +++ b/lib/tsan/tests/CMakeLists.txt @@ -14,6 +14,12 @@ set(TSAN_UNITTEST_CFLAGS -DGTEST_HAS_RTTI=0) set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH}) + +set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) +foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES}) + list(APPEND LINK_FLAGS -l${lib}) +endforeach() + if(APPLE) # Create a static library for test dependencies. @@ -33,12 +39,13 @@ if(APPLE) darwin_filter_host_archs(TSAN_SUPPORTED_ARCH TSAN_TEST_ARCH) list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS}) - set(LINK_FLAGS "-lc++") list(APPEND LINK_FLAGS ${DARWIN_osx_LINK_FLAGS}) add_weak_symbols("ubsan" LINK_FLAGS) add_weak_symbols("sanitizer_common" LINK_FLAGS) else() - set(LINK_FLAGS "-fsanitize=thread;-lstdc++;-lm") + list(APPEND LINK_FLAGS -fsanitize=thread) + list(APPEND LINK_FLAGS -lm) + list(APPEND LINK_FLAGS ${COMPILER_RT_TEST_LIBDISPATCH_CFLAGS}) endif() set(TSAN_RTL_HEADERS) diff --git a/lib/tsan/tests/rtl/tsan_bench.cc b/lib/tsan/tests/rtl/tsan_bench.cc index a3cf22f2c..013510114 100644 --- a/lib/tsan/tests/rtl/tsan_bench.cc +++ b/lib/tsan/tests/rtl/tsan_bench.cc @@ -1,9 +1,8 @@ //===-- tsan_bench.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_mop.cc b/lib/tsan/tests/rtl/tsan_mop.cc index f21742825..a5b0bdda3 100644 --- a/lib/tsan/tests/rtl/tsan_mop.cc +++ b/lib/tsan/tests/rtl/tsan_mop.cc @@ -1,9 +1,8 @@ //===-- tsan_mop.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_mutex.cc b/lib/tsan/tests/rtl/tsan_mutex.cc index 4d9c77961..af12e20b1 100644 --- a/lib/tsan/tests/rtl/tsan_mutex.cc +++ b/lib/tsan/tests/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_posix.cc b/lib/tsan/tests/rtl/tsan_posix.cc index e66dab609..d1940452c 100644 --- a/lib/tsan/tests/rtl/tsan_posix.cc +++ b/lib/tsan/tests/rtl/tsan_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_posix.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_posix_util.h b/lib/tsan/tests/rtl/tsan_posix_util.h index 340693ebb..e80039bfb 100644 --- a/lib/tsan/tests/rtl/tsan_posix_util.h +++ b/lib/tsan/tests/rtl/tsan_posix_util.h @@ -1,9 +1,8 @@ //===-- tsan_posix_util.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_string.cc b/lib/tsan/tests/rtl/tsan_string.cc index 75adc6c85..b236d4631 100644 --- a/lib/tsan/tests/rtl/tsan_string.cc +++ b/lib/tsan/tests/rtl/tsan_string.cc @@ -1,9 +1,8 @@ //===-- tsan_string.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test.cc b/lib/tsan/tests/rtl/tsan_test.cc index 842b41750..51a3b2731 100644 --- a/lib/tsan/tests/rtl/tsan_test.cc +++ b/lib/tsan/tests/rtl/tsan_test.cc @@ -1,9 +1,8 @@ //===-- tsan_test.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test_util.h b/lib/tsan/tests/rtl/tsan_test_util.h index 31b1b188f..df535150a 100644 --- a/lib/tsan/tests/rtl/tsan_test_util.h +++ b/lib/tsan/tests/rtl/tsan_test_util.h @@ -1,9 +1,8 @@ //===-- tsan_test_util.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test_util_posix.cc b/lib/tsan/tests/rtl/tsan_test_util_posix.cc index d00e26dd5..767c8294d 100644 --- a/lib/tsan/tests/rtl/tsan_test_util_posix.cc +++ b/lib/tsan/tests/rtl/tsan_test_util_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_test_util_posix.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_thread.cc b/lib/tsan/tests/rtl/tsan_thread.cc index 5646415a7..9e2da912e 100644 --- a/lib/tsan/tests/rtl/tsan_thread.cc +++ b/lib/tsan/tests/rtl/tsan_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_thread.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc index f6230e1be..43d0a012c 100644 --- a/lib/tsan/tests/unit/tsan_clock_test.cc +++ b/lib/tsan/tests/unit/tsan_clock_test.cc @@ -1,9 +1,8 @@ //===-- tsan_clock_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_dense_alloc_test.cc b/lib/tsan/tests/unit/tsan_dense_alloc_test.cc index e848e48c6..8c544b0d7 100644 --- a/lib/tsan/tests/unit/tsan_dense_alloc_test.cc +++ b/lib/tsan/tests/unit/tsan_dense_alloc_test.cc @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_flags_test.cc b/lib/tsan/tests/unit/tsan_flags_test.cc index aa8a024f9..ebdb28f6d 100644 --- a/lib/tsan/tests/unit/tsan_flags_test.cc +++ b/lib/tsan/tests/unit/tsan_flags_test.cc @@ -1,9 +1,8 @@ //===-- tsan_flags_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mman_test.cc b/lib/tsan/tests/unit/tsan_mman_test.cc index 26e13a55c..b2789b70f 100644 --- a/lib/tsan/tests/unit/tsan_mman_test.cc +++ b/lib/tsan/tests/unit/tsan_mman_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mman_test.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mutex_test.cc b/lib/tsan/tests/unit/tsan_mutex_test.cc index cce7f073b..0649c8aa5 100644 --- a/lib/tsan/tests/unit/tsan_mutex_test.cc +++ b/lib/tsan/tests/unit/tsan_mutex_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mutexset_test.cc b/lib/tsan/tests/unit/tsan_mutexset_test.cc index 335a7748c..5862138a8 100644 --- a/lib/tsan/tests/unit/tsan_mutexset_test.cc +++ b/lib/tsan/tests/unit/tsan_mutexset_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset_test.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_shadow_test.cc b/lib/tsan/tests/unit/tsan_shadow_test.cc index 17b17977b..21a4ddf97 100644 --- a/lib/tsan/tests/unit/tsan_shadow_test.cc +++ b/lib/tsan/tests/unit/tsan_shadow_test.cc @@ -1,9 +1,8 @@ //===-- tsan_shadow_test.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_stack_test.cc b/lib/tsan/tests/unit/tsan_stack_test.cc index 92e035d8d..d8b81d331 100644 --- a/lib/tsan/tests/unit/tsan_stack_test.cc +++ b/lib/tsan/tests/unit/tsan_stack_test.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_sync_test.cc b/lib/tsan/tests/unit/tsan_sync_test.cc index 801665478..7ea2826e8 100644 --- a/lib/tsan/tests/unit/tsan_sync_test.cc +++ b/lib/tsan/tests/unit/tsan_sync_test.cc @@ -1,9 +1,8 @@ //===-- tsan_sync_test.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_unit_test_main.cc b/lib/tsan/tests/unit/tsan_unit_test_main.cc index 2d55747a0..a1487310f 100644 --- a/lib/tsan/tests/unit/tsan_unit_test_main.cc +++ b/lib/tsan/tests/unit/tsan_unit_test_main.cc @@ -1,9 +1,8 @@ //===-- tsan_unit_test_main.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // |