diff options
Diffstat (limited to 'lib')
1104 files changed, 21373 insertions, 18250 deletions
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index b3731f653..39082aa06 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -17,6 +17,10 @@ if(COMPILER_RT_BUILD_BUILTINS) add_subdirectory(builtins) endif() +if(COMPILER_RT_BUILD_CRT AND COMPILER_RT_HAS_CRT) + add_subdirectory(crt) +endif() + function(compiler_rt_build_runtime runtime) string(TOUPPER ${runtime} runtime_uppercase) if(COMPILER_RT_HAS_${runtime_uppercase}) @@ -24,6 +28,9 @@ function(compiler_rt_build_runtime runtime) if(${runtime} STREQUAL tsan) add_subdirectory(tsan/dd) endif() + if(${runtime} STREQUAL scudo) + add_subdirectory(scudo/standalone) + endif() endif() endfunction() diff --git a/lib/asan/CMakeLists.txt b/lib/asan/CMakeLists.txt index 726da27d0..3c4434690 100644 --- a/lib/asan/CMakeLists.txt +++ b/lib/asan/CMakeLists.txt @@ -32,6 +32,10 @@ set(ASAN_SOURCES asan_thread.cc asan_win.cc) +if (NOT WIN32 AND NOT APPLE) + list(APPEND ASAN_SOURCES asan_interceptors_vfork.S) +endif() + set(ASAN_CXX_SOURCES asan_new_delete.cc) @@ -92,7 +96,7 @@ append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC -ftls-model=initial-exec ASAN_DYNAMIC_CFLAGS) append_list_if(MSVC /DEBUG ASAN_DYNAMIC_LINK_FLAGS) -set(ASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(ASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl ASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBRT rt ASAN_DYNAMIC_LIBS) diff --git a/lib/asan/asan_activation.cc b/lib/asan/asan_activation.cc index d642be934..fc97cbb55 100644 --- a/lib/asan/asan_activation.cc +++ b/lib/asan/asan_activation.cc @@ -1,9 +1,8 @@ //===-- asan_activation.cc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_activation.h b/lib/asan/asan_activation.h index d5e1ce433..93c290c2a 100644 --- a/lib/asan/asan_activation.h +++ b/lib/asan/asan_activation.h @@ -1,9 +1,8 @@ //===-- asan_activation.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_activation_flags.inc b/lib/asan/asan_activation_flags.inc index 1c66e5bb5..e0fdffc82 100644 --- a/lib/asan/asan_activation_flags.inc +++ b/lib/asan/asan_activation_flags.inc @@ -1,9 +1,8 @@ //===-- asan_activation_flags.inc -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc index c0fad4fa0..2ca6220d8 100644 --- a/lib/asan/asan_allocator.cc +++ b/lib/asan/asan_allocator.cc @@ -1,9 +1,8 @@ //===-- asan_allocator.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -880,6 +879,17 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); } +void *asan_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return asan_realloc(p, nmemb * size, stack); +} + void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { if (!p) return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); diff --git a/lib/asan/asan_allocator.h b/lib/asan/asan_allocator.h index c9b37dc7a..6add47be2 100644 --- a/lib/asan/asan_allocator.h +++ b/lib/asan/asan_allocator.h @@ -1,9 +1,8 @@ //===-- asan_allocator.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -134,11 +133,15 @@ const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x2000000000ULL; // 128G. typedef VeryCompactSizeClassMap SizeClassMap; # elif defined(__aarch64__) -// AArch64/SANITIZER_CAN_USER_ALLOCATOR64 is only for 42-bit VMA +// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA // so no need to different values for different VMA. const uptr kAllocatorSpace = 0x10000000000ULL; const uptr kAllocatorSize = 0x10000000000ULL; // 3T. typedef DefaultSizeClassMap SizeClassMap; +#elif defined(__sparc__) +const uptr kAllocatorSpace = ~(uptr)0; +const uptr kAllocatorSize = 0x20000000000ULL; // 2T. +typedef DefaultSizeClassMap SizeClassMap; # elif SANITIZER_WINDOWS const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x8000000000ULL; // 500G @@ -163,16 +166,6 @@ template <typename AddressSpaceView> using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #else // Fallback to SizeClassAllocator32. -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -# if SANITIZER_WORDSIZE == 32 -template <typename AddressSpaceView> -using ByteMapASVT = FlatByteMap<kNumRegions, AddressSpaceView>; -# elif SANITIZER_WORDSIZE == 64 -template <typename AddressSpaceView> -using ByteMapASVT = - TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; -# endif typedef CompactSizeClassMap SizeClassMap; template <typename AddressSpaceViewTy> struct AP32 { @@ -180,9 +173,8 @@ struct AP32 { static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 16; typedef __asan::SizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __asan::kRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = __asan::ByteMapASVT<AddressSpaceView>; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -192,21 +184,12 @@ using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif // SANITIZER_CAN_USE_ALLOCATOR64 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; -template <typename AddressSpaceView> -using AllocatorCacheASVT = - SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; -using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; template <typename AddressSpaceView> -using SecondaryAllocatorASVT = - LargeMmapAllocator<AsanMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, - AddressSpaceView>; -template <typename AddressSpaceView> using AsanAllocatorASVT = - CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, - AllocatorCacheASVT<AddressSpaceView>, - SecondaryAllocatorASVT<AddressSpaceView>>; + CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; using AsanAllocator = AsanAllocatorASVT<LocalAddressSpaceView>; +using AllocatorCache = AsanAllocator::AllocatorCache; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; @@ -226,6 +209,8 @@ void asan_delete(void *ptr, uptr size, uptr alignment, void *asan_malloc(uptr size, BufferedStackTrace *stack); void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack); +void *asan_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack); void *asan_valloc(uptr size, BufferedStackTrace *stack); void *asan_pvalloc(uptr size, BufferedStackTrace *stack); diff --git a/lib/asan/asan_debugging.cc b/lib/asan/asan_debugging.cc index 7c877ded3..7052a371e 100644 --- a/lib/asan/asan_debugging.cc +++ b/lib/asan/asan_debugging.cc @@ -1,9 +1,8 @@ //===-- asan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_descriptions.cc b/lib/asan/asan_descriptions.cc index cdb562d97..9b1217a86 100644 --- a/lib/asan/asan_descriptions.cc +++ b/lib/asan/asan_descriptions.cc @@ -1,9 +1,8 @@ //===-- asan_descriptions.cc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_descriptions.h b/lib/asan/asan_descriptions.h index 5c2d7662a..0226d844a 100644 --- a/lib/asan/asan_descriptions.h +++ b/lib/asan/asan_descriptions.h @@ -1,9 +1,8 @@ //===-- asan_descriptions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_errors.cc b/lib/asan/asan_errors.cc index 0ecd30dca..d598e37b9 100644 --- a/lib/asan/asan_errors.cc +++ b/lib/asan/asan_errors.cc @@ -1,9 +1,8 @@ //===-- asan_errors.cc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,7 +35,7 @@ static void OnStackUnwind(const SignalContext &sig, // corresponding code in the sanitizer_common and we use this callback to // print it. static_cast<const ScarinessScoreBase *>(callback_context)->Print(); - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, fast); } void ErrorDeadlySignal::Print() { @@ -178,6 +177,19 @@ void ErrorCallocOverflow::Print() { ReportErrorSummary(scariness.GetDescription(), stack); } +void ErrorReallocArrayOverflow::Print() { + Decorator d; + Printf("%s", d.Error()); + Report( + "ERROR: AddressSanitizer: reallocarray parameters overflow: count * size " + "(%zd * %zd) cannot be represented in type size_t (thread %s)\n", + count, size, AsanThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(scariness.GetDescription(), stack); +} + void ErrorPvallocOverflow::Print() { Decorator d; Printf("%s", d.Error()); diff --git a/lib/asan/asan_errors.h b/lib/asan/asan_errors.h index 7ddd7e94e..f0939ae32 100644 --- a/lib/asan/asan_errors.h +++ b/lib/asan/asan_errors.h @@ -1,9 +1,8 @@ //===-- asan_errors.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -164,6 +163,21 @@ struct ErrorCallocOverflow : ErrorBase { void Print(); }; +struct ErrorReallocArrayOverflow : ErrorBase { + const BufferedStackTrace *stack; + uptr count; + uptr size; + + ErrorReallocArrayOverflow() = default; // (*) + ErrorReallocArrayOverflow(u32 tid, BufferedStackTrace *stack_, uptr count_, + uptr size_) + : ErrorBase(tid, 10, "reallocarray-overflow"), + stack(stack_), + count(count_), + size(size_) {} + void Print(); +}; + struct ErrorPvallocOverflow : ErrorBase { const BufferedStackTrace *stack; uptr size; @@ -372,6 +386,7 @@ struct ErrorGeneric : ErrorBase { macro(MallocUsableSizeNotOwned) \ macro(SanitizerGetAllocatedSizeNotOwned) \ macro(CallocOverflow) \ + macro(ReallocArrayOverflow) \ macro(PvallocOverflow) \ macro(InvalidAllocationAlignment) \ macro(InvalidAlignedAllocAlignment) \ diff --git a/lib/asan/asan_fake_stack.cc b/lib/asan/asan_fake_stack.cc index 1c6184e3c..f8e1ac4b7 100644 --- a/lib/asan/asan_fake_stack.cc +++ b/lib/asan/asan_fake_stack.cc @@ -1,9 +1,8 @@ //===-- asan_fake_stack.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_fake_stack.h b/lib/asan/asan_fake_stack.h index da9a91c23..59ba85218 100644 --- a/lib/asan/asan_fake_stack.h +++ b/lib/asan/asan_fake_stack.h @@ -1,9 +1,8 @@ //===-- asan_fake_stack.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_flags.cc b/lib/asan/asan_flags.cc index 5682ab4eb..69d2a4088 100644 --- a/lib/asan/asan_flags.cc +++ b/lib/asan/asan_flags.cc @@ -1,9 +1,8 @@ //===-- asan_flags.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_flags.h b/lib/asan/asan_flags.h index 4935161c3..b55c81f07 100644 --- a/lib/asan/asan_flags.h +++ b/lib/asan/asan_flags.h @@ -1,9 +1,8 @@ //===-- asan_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_flags.inc b/lib/asan/asan_flags.inc index a9c97d53b..3c30a6283 100644 --- a/lib/asan/asan_flags.inc +++ b/lib/asan/asan_flags.inc @@ -1,9 +1,8 @@ //===-- asan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_fuchsia.cc b/lib/asan/asan_fuchsia.cc index 34399c923..aebc17f38 100644 --- a/lib/asan/asan_fuchsia.cc +++ b/lib/asan/asan_fuchsia.cc @@ -1,9 +1,8 @@ //===-- asan_fuchsia.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // @@ -179,7 +178,7 @@ static void ThreadStartHook(void *hook, uptr os_id) { SetCurrentThread(thread); // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false, + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, nullptr); } diff --git a/lib/asan/asan_globals.cc b/lib/asan/asan_globals.cc index 146234ac6..8b2fdb214 100644 --- a/lib/asan/asan_globals.cc +++ b/lib/asan/asan_globals.cc @@ -1,9 +1,8 @@ //===-- asan_globals.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -116,7 +115,11 @@ int GetGlobalsForAddress(uptr addr, Global *globals, u32 *reg_sites, if (flags()->report_globals >= 2) ReportGlobal(g, "Search"); if (IsAddressNearGlobal(addr, g)) { +#if defined(__GNUC__) && defined(__sparc__) + internal_memcpy(&globals[res], &g, sizeof(g)); +#else globals[res] = g; +#endif if (reg_sites) reg_sites[res] = FindRegistrationSite(&g); res++; diff --git a/lib/asan/asan_globals_win.cc b/lib/asan/asan_globals_win.cc index 0e75992bf..bdce37f70 100644 --- a/lib/asan/asan_globals_win.cc +++ b/lib/asan/asan_globals_win.cc @@ -1,9 +1,8 @@ //===-- asan_globals_win.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_init_version.h b/lib/asan/asan_init_version.h index c49fcd740..b806d794e 100644 --- a/lib/asan/asan_init_version.h +++ b/lib/asan/asan_init_version.h @@ -1,9 +1,8 @@ //===-- asan_init_version.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors.cc b/lib/asan/asan_interceptors.cc index aac2bb8a6..234cabce1 100644 --- a/lib/asan/asan_interceptors.cc +++ b/lib/asan/asan_interceptors.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -580,6 +579,11 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, } #endif // ASAN_INTERCEPT___CXA_ATEXIT +#if ASAN_INTERCEPT_VFORK +DEFINE_REAL(int, vfork); +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork); +#endif + // ---------------------- InitializeAsanInterceptors ---------------- {{{1 namespace __asan { void InitializeAsanInterceptors() { @@ -657,6 +661,10 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(__cxa_atexit); #endif +#if ASAN_INTERCEPT_VFORK + ASAN_INTERCEPT_FUNC(vfork); +#endif + InitializePlatformInterceptors(); VReport(1, "AddressSanitizer: libc interceptors initialized\n"); diff --git a/lib/asan/asan_interceptors.h b/lib/asan/asan_interceptors.h index 50895b167..2792e2f14 100644 --- a/lib/asan/asan_interceptors.h +++ b/lib/asan/asan_interceptors.h @@ -1,9 +1,8 @@ //===-- asan_interceptors.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -106,6 +105,13 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___STRDUP 0 #endif +#if SANITIZER_LINUX && (defined(__arm__) || defined(__aarch64__) || \ + defined(__i386__) || defined(__x86_64__)) +# define ASAN_INTERCEPT_VFORK 1 +#else +# define ASAN_INTERCEPT_VFORK 0 +#endif + DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) DECLARE_REAL(char*, strchr, const char *str, int c) DECLARE_REAL(SIZE_T, strlen, const char *s) @@ -116,12 +122,12 @@ DECLARE_REAL(char*, strstr, const char *s1, const char *s2) #if !SANITIZER_MAC #define ASAN_INTERCEPT_FUNC(name) \ do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ + if (!INTERCEPT_FUNCTION(name)) \ VReport(1, "AddressSanitizer: failed to intercept '" #name "'\n"); \ } while (0) #define ASAN_INTERCEPT_FUNC_VER(name, ver) \ do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ VReport( \ 1, "AddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ } while (0) diff --git a/lib/asan/asan_interceptors_memintrinsics.cc b/lib/asan/asan_interceptors_memintrinsics.cc index 39e32cdad..e17f9ba4a 100644 --- a/lib/asan/asan_interceptors_memintrinsics.cc +++ b/lib/asan/asan_interceptors_memintrinsics.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors_memintrinsics.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors_memintrinsics.h b/lib/asan/asan_interceptors_memintrinsics.h index a071e8f68..1fd65fe24 100644 --- a/lib/asan/asan_interceptors_memintrinsics.h +++ b/lib/asan/asan_interceptors_memintrinsics.h @@ -1,9 +1,8 @@ //===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors_vfork.S b/lib/asan/asan_interceptors_vfork.S new file mode 100644 index 000000000..90a169d4b --- /dev/null +++ b/lib/asan/asan_interceptors_vfork.S @@ -0,0 +1,12 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) +#define COMMON_INTERCEPTOR_SPILL_AREA __asan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __asan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/asan/asan_interface.inc b/lib/asan/asan_interface.inc index e65f61722..7c341f22e 100644 --- a/lib/asan/asan_interface.inc +++ b/lib/asan/asan_interface.inc @@ -1,9 +1,8 @@ //===-- asan_interface.inc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Asan interface list. @@ -39,6 +38,7 @@ INTERFACE_FUNCTION(__asan_get_report_pc) INTERFACE_FUNCTION(__asan_get_report_sp) INTERFACE_FUNCTION(__asan_get_shadow_mapping) INTERFACE_FUNCTION(__asan_handle_no_return) +INTERFACE_FUNCTION(__asan_handle_vfork) INTERFACE_FUNCTION(__asan_init) INTERFACE_FUNCTION(__asan_load_cxx_array_cookie) INTERFACE_FUNCTION(__asan_load1) diff --git a/lib/asan/asan_interface_internal.h b/lib/asan/asan_interface_internal.h index b974c0cc4..c83aa11d7 100644 --- a/lib/asan/asan_interface_internal.h +++ b/lib/asan/asan_interface_internal.h @@ -1,9 +1,8 @@ //===-- asan_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -250,6 +249,8 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char* __asan_default_suppressions(); + + SANITIZER_INTERFACE_ATTRIBUTE void __asan_handle_vfork(void *sp); } // extern "C" #endif // ASAN_INTERFACE_INTERNAL_H diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h index 57869497c..e4f771079 100644 --- a/lib/asan/asan_internal.h +++ b/lib/asan/asan_internal.h @@ -1,9 +1,8 @@ //===-- asan_internal.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc index a150b1955..f91823289 100644 --- a/lib/asan/asan_linux.cc +++ b/lib/asan/asan_linux.cc @@ -1,9 +1,8 @@ //===-- asan_linux.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc index 17a0ec577..e776acd2f 100644 --- a/lib/asan/asan_mac.cc +++ b/lib/asan/asan_mac.cc @@ -1,9 +1,8 @@ //===-- asan_mac.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -182,8 +181,8 @@ void asan_register_worker_thread(int parent_tid, StackTrace *stack) { t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr, parent_tid, stack, /* detached */ true); t->Init(); - asanThreadRegistry().StartThread(t->tid(), GetTid(), - /* workerthread */ true, 0); + asanThreadRegistry().StartThread(t->tid(), GetTid(), ThreadType::Worker, + nullptr); SetCurrentThread(t); } } diff --git a/lib/asan/asan_malloc_linux.cc b/lib/asan/asan_malloc_linux.cc index 0a534fe21..86fcd3b12 100644 --- a/lib/asan/asan_malloc_linux.cc +++ b/lib/asan/asan_malloc_linux.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_linux.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -166,6 +165,14 @@ INTERCEPTOR(void*, realloc, void *ptr, uptr size) { return asan_realloc(ptr, size, &stack); } +#if SANITIZER_INTERCEPT_REALLOCARRAY +INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) { + ENSURE_ASAN_INITED(); + GET_STACK_TRACE_MALLOC; + return asan_reallocarray(ptr, nmemb, size, &stack); +} +#endif // SANITIZER_INTERCEPT_REALLOCARRAY + #if SANITIZER_INTERCEPT_MEMALIGN INTERCEPTOR(void*, memalign, uptr boundary, uptr size) { GET_STACK_TRACE_MALLOC; diff --git a/lib/asan/asan_malloc_local.h b/lib/asan/asan_malloc_local.h index 0e8de207d..65e3e5a97 100644 --- a/lib/asan/asan_malloc_local.h +++ b/lib/asan/asan_malloc_local.h @@ -1,9 +1,8 @@ //===-- asan_malloc_local.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_malloc_mac.cc b/lib/asan/asan_malloc_mac.cc index 27281f1bc..06dc1c289 100644 --- a/lib/asan/asan_malloc_mac.cc +++ b/lib/asan/asan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "asan_report.h" #include "asan_stack.h" #include "asan_stats.h" +#include "lsan/lsan_common.h" using namespace __asan; #define COMMON_MALLOC_ZONE_NAME "asan" @@ -58,10 +58,13 @@ using namespace __asan; GET_STACK_TRACE_FREE; \ ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); #define COMMON_MALLOC_NAMESPACE __asan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 1 #include "sanitizer_common/sanitizer_malloc_mac.inc" namespace COMMON_MALLOC_NAMESPACE { + bool HandleDlopenInit() { static_assert(SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be true"); @@ -82,4 +85,18 @@ bool HandleDlopenInit() { } } // namespace COMMON_MALLOC_NAMESPACE +namespace { + +void mi_extra_init(sanitizer_malloc_introspection_t *mi) { + uptr last_byte_plus_one = 0; + mi->allocator_ptr = 0; + // Range is [begin_ptr, end_ptr) + __lsan::GetAllocatorGlobalRange(&(mi->allocator_ptr), &last_byte_plus_one); + CHECK_NE(mi->allocator_ptr, 0); + CHECK_GT(last_byte_plus_one, mi->allocator_ptr); + mi->allocator_size = last_byte_plus_one - (mi->allocator_ptr); + CHECK_GT(mi->allocator_size, 0); +} +} // namespace + #endif diff --git a/lib/asan/asan_malloc_win.cc b/lib/asan/asan_malloc_win.cc index 887936431..b13d798a3 100644 --- a/lib/asan/asan_malloc_win.cc +++ b/lib/asan/asan_malloc_win.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_win.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,6 +48,18 @@ using namespace __asan; // NOLINT extern "C" { ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize(void *ptr) { + GET_CURRENT_PC_BP_SP; + (void)sp; + return asan_malloc_usable_size(ptr, pc, bp); +} + +ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize_base(void *ptr) { + return _msize(ptr); +} + +ALLOCATION_FUNCTION_ATTRIBUTE void free(void *ptr) { GET_STACK_TRACE_FREE; return asan_free(ptr, &stack, FROM_MALLOC); @@ -125,7 +136,16 @@ void *_recalloc(void *p, size_t n, size_t elem_size) { const size_t size = n * elem_size; if (elem_size != 0 && size / elem_size != n) return 0; - return realloc(p, size); + + size_t old_size = _msize(p); + void *new_alloc = malloc(size); + if (new_alloc) { + REAL(memcpy)(new_alloc, p, Min(size, old_size)); + if (old_size < size) + REAL(memset)(((u8 *)new_alloc) + old_size, 0, size - old_size); + free(p); + } + return new_alloc; } ALLOCATION_FUNCTION_ATTRIBUTE @@ -134,18 +154,6 @@ void *_recalloc_base(void *p, size_t n, size_t elem_size) { } ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize(void *ptr) { - GET_CURRENT_PC_BP_SP; - (void)sp; - return asan_malloc_usable_size(ptr, pc, bp); -} - -ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize_base(void *ptr) { - return _msize(ptr); -} - -ALLOCATION_FUNCTION_ATTRIBUTE void *_expand(void *memblock, size_t size) { // _expand is used in realloc-like functions to resize the buffer if possible. // We don't want memory to stand still while resizing buffers, so return 0. @@ -199,11 +207,31 @@ INTERCEPTOR_WINAPI(BOOL, HeapFree, HANDLE hHeap, DWORD dwFlags, LPVOID lpMem) { INTERCEPTOR_WINAPI(LPVOID, HeapReAlloc, HANDLE hHeap, DWORD dwFlags, LPVOID lpMem, SIZE_T dwBytes) { GET_STACK_TRACE_MALLOC; + GET_CURRENT_PC_BP_SP; + (void)sp; // Realloc should never reallocate in place. if (dwFlags & HEAP_REALLOC_IN_PLACE_ONLY) return nullptr; CHECK(dwFlags == 0 && "unsupported heap flags"); - return asan_realloc(lpMem, dwBytes, &stack); + // HeapReAlloc and HeapAlloc both happily accept 0 sized allocations. + // passing a 0 size into asan_realloc will free the allocation. + // To avoid this and keep behavior consistent, fudge the size if 0. + // (asan_malloc already does this) + if (dwBytes == 0) + dwBytes = 1; + size_t old_size; + if (dwFlags & HEAP_ZERO_MEMORY) + old_size = asan_malloc_usable_size(lpMem, pc, bp); + void *ptr = asan_realloc(lpMem, dwBytes, &stack); + if (ptr == nullptr) + return nullptr; + + if (dwFlags & HEAP_ZERO_MEMORY) { + size_t new_size = asan_malloc_usable_size(ptr, pc, bp); + if (old_size < new_size) + REAL(memset)(((u8 *)ptr) + old_size, 0, new_size - old_size); + } + return ptr; } INTERCEPTOR_WINAPI(SIZE_T, HeapSize, HANDLE hHeap, DWORD dwFlags, diff --git a/lib/asan/asan_mapping.h b/lib/asan/asan_mapping.h index f3696da62..2a32a23e1 100644 --- a/lib/asan/asan_mapping.h +++ b/lib/asan/asan_mapping.h @@ -1,9 +1,8 @@ //===-- asan_mapping.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -101,6 +100,13 @@ // || `[0x10000000000000, 0x11ffffffffffff]` || LowShadow || // || `[0x00000000000000, 0x0fffffffffffff]` || LowMem || // +// Default Linux/SPARC64 (52-bit VMA) mapping: +// || `[0x8000000000000, 0xfffffffffffff]` || HighMem || +// || `[0x1080000000000, 0x207ffffffffff]` || HighShadow || +// || `[0x0090000000000, 0x107ffffffffff]` || ShadowGap || +// || `[0x0080000000000, 0x008ffffffffff]` || LowShadow || +// || `[0x0000000000000, 0x007ffffffffff]` || LowMem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -163,6 +169,7 @@ static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; +static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000 static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 @@ -225,6 +232,8 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; # define SHADOW_OFFSET kDefaultShadowOffset64 # elif defined(__mips64) # define SHADOW_OFFSET kMIPS64_ShadowOffset64 +#elif defined(__sparc__) +#define SHADOW_OFFSET kSPARC64_ShadowOffset64 # elif SANITIZER_WINDOWS64 # define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # else @@ -271,6 +280,8 @@ extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init. #if SANITIZER_MYRIAD2 #include "asan_mapping_myriad.h" +#elif defined(__sparc__) && SANITIZER_WORDSIZE == 64 +#include "asan_mapping_sparc64.h" #else #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) diff --git a/lib/asan/asan_mapping_myriad.h b/lib/asan/asan_mapping_myriad.h index baa3247bd..6969e3a49 100644 --- a/lib/asan/asan_mapping_myriad.h +++ b/lib/asan/asan_mapping_myriad.h @@ -1,9 +1,8 @@ //===-- asan_mapping_myriad.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_mapping_sparc64.h b/lib/asan/asan_mapping_sparc64.h new file mode 100644 index 000000000..432a1816f --- /dev/null +++ b/lib/asan/asan_mapping_sparc64.h @@ -0,0 +1,101 @@ +//===-- asan_mapping_sparc64.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// SPARC64-specific definitions for ASan memory mapping. +//===----------------------------------------------------------------------===// +#ifndef ASAN_MAPPING_SPARC64_H +#define ASAN_MAPPING_SPARC64_H + +// This is tailored to the 52-bit VM layout on SPARC-T4 and later. +// The VM space is split into two 51-bit halves at both ends: the low part +// has all the bits above the 51st cleared, while the high part has them set. +// 0xfff8000000000000 - 0xffffffffffffffff +// 0x0000000000000000 - 0x0007ffffffffffff + +#define VMA_BITS 52 +#define HIGH_BITS (64 - VMA_BITS) + +// The idea is to chop the high bits before doing the scaling, so the two +// parts become contiguous again and the usual scheme can be applied. + +#define MEM_TO_SHADOW(mem) \ + ((((mem) << HIGH_BITS) >> (HIGH_BITS + (SHADOW_SCALE))) + (SHADOW_OFFSET)) + +#define kLowMemBeg 0 +#define kLowMemEnd (SHADOW_OFFSET - 1) + +#define kLowShadowBeg SHADOW_OFFSET +#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) + +// But of course there is the huge hole between the high shadow memory, +// which is in the low part, and the beginning of the high part. + +#define kHighMemBeg (-(1ULL << (VMA_BITS - 1))) + +#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) +#define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd) + +#define kMidShadowBeg 0 +#define kMidShadowEnd 0 + +// With the zero shadow base we can not actually map pages starting from 0. +// This constant is somewhat arbitrary. +#define kZeroBaseShadowStart 0 +#define kZeroBaseMaxShadowStart (1 << 18) + +#define kShadowGapBeg (kLowShadowEnd + 1) +#define kShadowGapEnd (kHighShadowBeg - 1) + +#define kShadowGap2Beg 0 +#define kShadowGap2End 0 + +#define kShadowGap3Beg 0 +#define kShadowGap3End 0 + +namespace __asan { + +static inline bool AddrIsInLowMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return a <= kLowMemEnd; +} + +static inline bool AddrIsInLowShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return a >= kLowShadowBeg && a <= kLowShadowEnd; +} + +static inline bool AddrIsInMidMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return false; +} + +static inline bool AddrIsInMidShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return false; +} + +static inline bool AddrIsInHighMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd; +} + +static inline bool AddrIsInHighShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd; +} + +static inline bool AddrIsInShadowGap(uptr a) { + PROFILE_ASAN_MAPPING(); + return a >= kShadowGapBeg && a <= kShadowGapEnd; +} + +} // namespace __asan + +#endif // ASAN_MAPPING_SPARC64_H diff --git a/lib/asan/asan_memory_profile.cc b/lib/asan/asan_memory_profile.cc index 8c86d9fe4..87d874d2f 100644 --- a/lib/asan/asan_memory_profile.cc +++ b/lib/asan/asan_memory_profile.cc @@ -1,9 +1,8 @@ //===-- asan_memory_profile.cc.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc index e6053c1fe..8b53ac96e 100644 --- a/lib/asan/asan_new_delete.cc +++ b/lib/asan/asan_new_delete.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_poisoning.cc b/lib/asan/asan_poisoning.cc index 1e9c37a13..44b87c76e 100644 --- a/lib/asan/asan_poisoning.cc +++ b/lib/asan/asan_poisoning.cc @@ -1,9 +1,8 @@ //===-- asan_poisoning.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_poisoning.h b/lib/asan/asan_poisoning.h index c94794cea..ca14d11fd 100644 --- a/lib/asan/asan_poisoning.h +++ b/lib/asan/asan_poisoning.h @@ -1,9 +1,8 @@ //===-- asan_poisoning.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_posix.cc b/lib/asan/asan_posix.cc index ca99c04b3..5c5e0359a 100644 --- a/lib/asan/asan_posix.cc +++ b/lib/asan/asan_posix.cc @@ -1,9 +1,8 @@ //===-- asan_posix.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_preinit.cc b/lib/asan/asan_preinit.cc index a3986d24f..444998c44 100644 --- a/lib/asan/asan_preinit.cc +++ b/lib/asan/asan_preinit.cc @@ -1,9 +1,8 @@ //===-- asan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_premap_shadow.cc b/lib/asan/asan_premap_shadow.cc index 229eba99f..6e547718c 100644 --- a/lib/asan/asan_premap_shadow.cc +++ b/lib/asan/asan_premap_shadow.cc @@ -1,9 +1,8 @@ //===-- asan_premap_shadow.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_premap_shadow.h b/lib/asan/asan_premap_shadow.h index 41acbdbbb..c9c886e8d 100644 --- a/lib/asan/asan_premap_shadow.h +++ b/lib/asan/asan_premap_shadow.h @@ -1,9 +1,8 @@ //===-- asan_mapping.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_report.cc b/lib/asan/asan_report.cc index 439712350..551753b4a 100644 --- a/lib/asan/asan_report.cc +++ b/lib/asan/asan_report.cc @@ -1,9 +1,8 @@ //===-- asan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -264,6 +263,13 @@ void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack) { in_report.ReportError(error); } +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorReallocArrayOverflow error(GetCurrentTidOrInvalid(), stack, count, size); + in_report.ReportError(error); +} + void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack) { ScopedInErrorReport in_report(/*fatal*/ true); ErrorPvallocOverflow error(GetCurrentTidOrInvalid(), stack, size); diff --git a/lib/asan/asan_report.h b/lib/asan/asan_report.h index b0c167dda..dcf60894e 100644 --- a/lib/asan/asan_report.h +++ b/lib/asan/asan_report.h @@ -1,9 +1,8 @@ //===-- asan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,6 +61,8 @@ void ReportMallocUsableSizeNotOwned(uptr addr, BufferedStackTrace *stack); void ReportSanitizerGetAllocatedSizeNotOwned(uptr addr, BufferedStackTrace *stack); void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack); +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack); void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack); void ReportInvalidAllocationAlignment(uptr alignment, BufferedStackTrace *stack); diff --git a/lib/asan/asan_rtems.cc b/lib/asan/asan_rtems.cc index b48cc6a75..4878f4d67 100644 --- a/lib/asan/asan_rtems.cc +++ b/lib/asan/asan_rtems.cc @@ -1,9 +1,8 @@ //===-- asan_rtems.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,8 +183,8 @@ static void ThreadStartHook(void *hook, uptr os_id) { // Determine whether we are starting or restarting the thread. if (status == ThreadStatusCreated) // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, - /*workerthread*/ false, nullptr); + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, + nullptr); else { // In a thread restart, a thread may resume execution at an // arbitrary function entry point, with its stack and TLS state diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc index 13344f3b8..db8dcd068 100644 --- a/lib/asan/asan_rtl.cc +++ b/lib/asan/asan_rtl.cc @@ -1,9 +1,8 @@ //===-- asan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -598,6 +597,19 @@ void NOINLINE __asan_handle_no_return() { curr_thread->fake_stack()->HandleNoReturn(); } +extern "C" void *__asan_extra_spill_area() { + AsanThread *t = GetCurrentThread(); + CHECK(t); + return t->extra_spill_area(); +} + +void __asan_handle_vfork(void *sp) { + AsanThread *t = GetCurrentThread(); + CHECK(t); + uptr bottom = t->stack_bottom(); + PoisonShadow(bottom, (uptr)sp - bottom, 0); +} + void NOINLINE __asan_set_death_callback(void (*callback)(void)) { SetUserDieCallback(callback); } diff --git a/lib/asan/asan_scariness_score.h b/lib/asan/asan_scariness_score.h index 7f095dd29..9e7ba47d8 100644 --- a/lib/asan/asan_scariness_score.h +++ b/lib/asan/asan_scariness_score.h @@ -1,9 +1,8 @@ //===-- asan_scariness_score.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_shadow_setup.cc b/lib/asan/asan_shadow_setup.cc index 083926e70..9cfa4e2bd 100644 --- a/lib/asan/asan_shadow_setup.cc +++ b/lib/asan/asan_shadow_setup.cc @@ -1,9 +1,8 @@ //===-- asan_shadow_setup.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_stack.cc b/lib/asan/asan_stack.cc index cf7a587fa..b244da4fa 100644 --- a/lib/asan/asan_stack.cc +++ b/lib/asan/asan_stack.cc @@ -1,9 +1,8 @@ //===-- asan_stack.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,8 +26,57 @@ u32 GetMallocContextSize() { return atomic_load(&malloc_context_size, memory_order_acquire); } +namespace { + +// ScopedUnwinding is a scope for stacktracing member of a context +class ScopedUnwinding { + public: + explicit ScopedUnwinding(AsanThread *t) : thread(t) { + if (thread) { + can_unwind = !thread->isUnwinding(); + thread->setUnwinding(true); + } + } + ~ScopedUnwinding() { + if (thread) + thread->setUnwinding(false); + } + + bool CanUnwind() const { return can_unwind; } + + private: + AsanThread *thread = nullptr; + bool can_unwind = true; +}; + +} // namespace + } // namespace __asan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __asan; + size = 0; + if (UNLIKELY(!asan_inited)) + return; + request_fast = StackTrace::WillUseFastUnwind(request_fast); + AsanThread *t = GetCurrentThread(); + ScopedUnwinding unwind_scope(t); + if (!unwind_scope.CanUnwind()) + return; + if (request_fast) { + if (t) { + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + true); + } + return; + } + if (SANITIZER_MIPS && t && + !IsValidFrame(bp, t->stack_top(), t->stack_bottom())) + return; + Unwind(max_depth, pc, bp, context, 0, 0, false); +} + // ------------------ Interface -------------- {{{1 extern "C" { diff --git a/lib/asan/asan_stack.h b/lib/asan/asan_stack.h index 8e9df888d..3a4b3cefc 100644 --- a/lib/asan/asan_stack.h +++ b/lib/asan/asan_stack.h @@ -1,9 +1,8 @@ //===-- asan_stack.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,34 +26,6 @@ static const u32 kDefaultMallocContextSize = 30; void SetMallocContextSize(u32 size); u32 GetMallocContextSize(); -// Get the stack trace with the given pc and bp. -// The pc will be in the position 0 of the resulting stack trace. -// The bp may refer to the current frame or to the caller's frame. -ALWAYS_INLINE -void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, - void *context, bool fast) { -#if SANITIZER_WINDOWS - stack->Unwind(max_depth, pc, bp, context, 0, 0, fast); -#else - AsanThread *t; - stack->size = 0; - if (LIKELY(asan_inited)) { - if ((t = GetCurrentThread()) && !t->isUnwinding()) { - uptr stack_top = t->stack_top(); - uptr stack_bottom = t->stack_bottom(); - ScopedUnwinding unwind_scope(t); - if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { - stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, - fast); - } - } else if (!t && !fast) { - /* If GetCurrentThread() has failed, try to do slow unwind anyways. */ - stack->Unwind(max_depth, pc, bp, context, 0, 0, false); - } - } -#endif // SANITIZER_WINDOWS -} - } // namespace __asan // NOTE: A Rule of thumb is to retrieve stack trace in the interceptors @@ -71,19 +42,19 @@ void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ } \ } else { \ - GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), 0, fast); \ + stack.Unwind(StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast, max_size); \ } #define GET_STACK_TRACE_FATAL(pc, bp) \ BufferedStackTrace stack; \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, 0, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind(pc, bp, nullptr, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_SIGNAL(sig) \ BufferedStackTrace stack; \ - GetStackTrace(&stack, kStackTraceMax, (sig).pc, (sig).bp, (sig).context, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind((sig).pc, (sig).bp, (sig).context, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/lib/asan/asan_stats.cc b/lib/asan/asan_stats.cc index b8c68c32d..2f996ce63 100644 --- a/lib/asan/asan_stats.cc +++ b/lib/asan/asan_stats.cc @@ -1,9 +1,8 @@ //===-- asan_stats.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_stats.h b/lib/asan/asan_stats.h index 4605135e1..d6da65340 100644 --- a/lib/asan/asan_stats.h +++ b/lib/asan/asan_stats.h @@ -1,9 +1,8 @@ //===-- asan_stats.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_suppressions.cc b/lib/asan/asan_suppressions.cc index ac8aa023f..118853e61 100644 --- a/lib/asan/asan_suppressions.cc +++ b/lib/asan/asan_suppressions.cc @@ -1,9 +1,8 @@ //===-- asan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_suppressions.h b/lib/asan/asan_suppressions.h index 5246b4b30..9bf297602 100644 --- a/lib/asan/asan_suppressions.h +++ b/lib/asan/asan_suppressions.h @@ -1,9 +1,8 @@ //===-- asan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc index 0895e4ce0..e63561c22 100644 --- a/lib/asan/asan_thread.cc +++ b/lib/asan/asan_thread.cc @@ -1,9 +1,8 @@ //===-- asan_thread.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -246,8 +245,7 @@ void AsanThread::Init(const InitOptions *options) { thread_return_t AsanThread::ThreadStart( tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); - asanThreadRegistry().StartThread(tid(), os_id, /*workerthread*/ false, - nullptr); + asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr); if (signal_thread_is_registered) atomic_store(signal_thread_is_registered, 1, memory_order_release); diff --git a/lib/asan/asan_thread.h b/lib/asan/asan_thread.h index 660919211..d725e8886 100644 --- a/lib/asan/asan_thread.h +++ b/lib/asan/asan_thread.h @@ -1,9 +1,8 @@ //===-- asan_thread.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -131,6 +130,8 @@ class AsanThread { AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } AsanStats &stats() { return stats_; } + void *extra_spill_area() { return &extra_spill_area_; } + private: // NOTE: There is no AsanThread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. @@ -166,18 +167,7 @@ class AsanThread { AsanThreadLocalMallocStorage malloc_storage_; AsanStats stats_; bool unwinding_; -}; - -// ScopedUnwinding is a scope for stacktracing member of a context -class ScopedUnwinding { - public: - explicit ScopedUnwinding(AsanThread *t) : thread(t) { - t->setUnwinding(true); - } - ~ScopedUnwinding() { thread->setUnwinding(false); } - - private: - AsanThread *thread; + uptr extra_spill_area_; }; // Returns a single instance of registry. diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc index 068f4a5d2..c9b81fa8f 100644 --- a/lib/asan/asan_win.cc +++ b/lib/asan/asan_win.cc @@ -1,9 +1,8 @@ //===-- asan_win.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -105,7 +104,9 @@ INTERCEPTOR_WINAPI(void, RaiseException, void *a, void *b, void *c, void *d) { #ifdef _WIN64 -INTERCEPTOR_WINAPI(int, __C_specific_handler, void *a, void *b, void *c, void *d) { // NOLINT +INTERCEPTOR_WINAPI(EXCEPTION_DISPOSITION, __C_specific_handler, + _EXCEPTION_RECORD *a, void *b, _CONTEXT *c, + _DISPATCHER_CONTEXT *d) { // NOLINT CHECK(REAL(__C_specific_handler)); __asan_handle_no_return(); return REAL(__C_specific_handler)(a, b, c, d); @@ -136,10 +137,9 @@ static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { return t->ThreadStart(GetTid(), /* signal_thread_is_registered */ nullptr); } -INTERCEPTOR_WINAPI(DWORD, CreateThread, - void* security, uptr stack_size, - DWORD (__stdcall *start_routine)(void*), void* arg, - DWORD thr_flags, void* tid) { +INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, + SIZE_T stack_size, LPTHREAD_START_ROUTINE start_routine, + void *arg, DWORD thr_flags, DWORD *tid) { // Strict init-order checking is thread-hostile. if (flags()->strict_init_order) StopInitOrderChecking(); @@ -149,9 +149,9 @@ INTERCEPTOR_WINAPI(DWORD, CreateThread, bool detached = false; // FIXME: how can we determine it on Windows? u32 current_tid = GetCurrentTidOrInvalid(); AsanThread *t = - AsanThread::Create(start_routine, arg, current_tid, &stack, detached); - return REAL(CreateThread)(security, stack_size, - asan_thread_start, t, thr_flags, tid); + AsanThread::Create(start_routine, arg, current_tid, &stack, detached); + return REAL(CreateThread)(security, stack_size, asan_thread_start, t, + thr_flags, tid); } // }}} @@ -355,6 +355,19 @@ __declspec(allocate(".CRT$XLAB")) void (NTAPI *__asan_tls_init)(void *, unsigned long, void *) = asan_thread_init; #endif +static void NTAPI asan_thread_exit(void *module, DWORD reason, void *reserved) { + if (reason == DLL_THREAD_DETACH) { + // Unpoison the thread's stack because the memory may be re-used. + NT_TIB *tib = (NT_TIB *)NtCurrentTeb(); + uptr stackSize = (uptr)tib->StackBase - (uptr)tib->StackLimit; + __asan_unpoison_memory_region(tib->StackLimit, stackSize); + } +} + +#pragma section(".CRT$XLY", long, read) // NOLINT +__declspec(allocate(".CRT$XLY")) void (NTAPI *__asan_tls_exit)(void *, + unsigned long, void *) = asan_thread_exit; + WIN_FORCE_LINK(__asan_dso_reg_hook) // }}} diff --git a/lib/asan/asan_win_dll_thunk.cc b/lib/asan/asan_win_dll_thunk.cc index df593ab92..47b3948c5 100644 --- a/lib/asan/asan_win_dll_thunk.cc +++ b/lib/asan/asan_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- asan_win_dll_thunk.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_win_dynamic_runtime_thunk.cc b/lib/asan/asan_win_dynamic_runtime_thunk.cc index 416c73b23..cf4a59842 100644 --- a/lib/asan/asan_win_dynamic_runtime_thunk.cc +++ b/lib/asan/asan_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- asan_win_dynamic_runtime_thunk.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_win_weak_interception.cc b/lib/asan/asan_win_weak_interception.cc index ca26f914c..19965ca47 100644 --- a/lib/asan/asan_win_weak_interception.cc +++ b/lib/asan/asan_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- asan_win_weak_interception.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Address Sanitizer when it is implemented as diff --git a/lib/asan/scripts/asan_device_setup b/lib/asan/scripts/asan_device_setup index 5e679e366..041bf92e6 100755 --- a/lib/asan/scripts/asan_device_setup +++ b/lib/asan/scripts/asan_device_setup @@ -1,10 +1,9 @@ #!/bin/bash #===- lib/asan/scripts/asan_device_setup -----------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # Prepare Android device to run ASan applications. # diff --git a/lib/asan/scripts/asan_symbolize.py b/lib/asan/scripts/asan_symbolize.py index 2dbb05283..4fb3355d7 100755 --- a/lib/asan/scripts/asan_symbolize.py +++ b/lib/asan/scripts/asan_symbolize.py @@ -1,26 +1,36 @@ #!/usr/bin/env python #===- lib/asan/scripts/asan_symbolize.py -----------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# +""" +Example of use: + asan_symbolize.py -c "$HOME/opt/cross/bin/arm-linux-gnueabi-" -s "$HOME/SymbolFiles" < asan.log + +PLUGINS + +This script provides a way for external plug-ins to hook into the behaviour of +various parts of this script (see `--plugins`). This is useful for situations +where it is necessary to handle site-specific quirks (e.g. binaries with debug +symbols only accessible via a remote service) without having to modify the +script itself. + +""" import argparse import bisect import getopt +import logging import os import re import subprocess import sys symbolizers = {} -DEBUG = False demangle = False binutils_prefix = None -sysroot_path = None -binary_name_filter = None fix_filename_patterns = None logfile = sys.stdin allow_system_symbolizer = True @@ -35,9 +45,6 @@ def fix_filename(file_name): file_name = re.sub('.*crtstuff.c:0', '???:0', file_name) return file_name -def sysroot_path_filter(binary_name): - return sysroot_path + binary_name - def is_valid_arch(s): return s in ["i386", "x86_64", "x86_64h", "arm", "armv6", "armv7", "armv7s", "armv7k", "arm64", "powerpc64", "powerpc64le", "s390x", "s390"] @@ -88,8 +95,7 @@ class LLVMSymbolizer(Symbolizer): if self.system == 'Darwin': for hint in self.dsym_hints: cmd.append('--dsym-hint=%s' % hint) - if DEBUG: - print(' '.join(cmd)) + logging.debug(' '.join(cmd)) try: result = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -106,8 +112,7 @@ class LLVMSymbolizer(Symbolizer): result = [] try: symbolizer_input = '"%s" %s' % (binary, offset) - if DEBUG: - print(symbolizer_input) + logging.debug(symbolizer_input) self.pipe.stdin.write("%s\n" % symbolizer_input) while True: function_name = self.pipe.stdout.readline().rstrip() @@ -152,8 +157,7 @@ class Addr2LineSymbolizer(Symbolizer): if demangle: cmd += ['--demangle'] cmd += ['-e', self.binary] - if DEBUG: - print(' '.join(cmd)) + logging.debug(' '.join(cmd)) return subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0, @@ -222,8 +226,7 @@ class DarwinSymbolizer(Symbolizer): self.open_atos() def open_atos(self): - if DEBUG: - print('atos -o %s -arch %s' % (self.binary, self.arch)) + logging.debug('atos -o %s -arch %s', self.binary, self.arch) cmdline = ['atos', '-o', self.binary, '-arch', self.arch] self.atos = UnbufferedLineConverter(cmdline, close_stderr=True) @@ -241,8 +244,7 @@ class DarwinSymbolizer(Symbolizer): # A well-formed atos response looks like this: # foo(type1, type2) (in object.name) (filename.cc:80) match = re.match('^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line) - if DEBUG: - print('atos_line: ', atos_line) + logging.debug('atos_line: %s', atos_line) if match: function_name = match.group(1) function_name = re.sub('\(.*?\)', '', function_name) @@ -363,7 +365,8 @@ class BreakpadSymbolizer(Symbolizer): class SymbolizationLoop(object): - def __init__(self, binary_name_filter=None, dsym_hint_producer=None): + def __init__(self, plugin_proxy=None, dsym_hint_producer=None): + self.plugin_proxy = plugin_proxy if sys.platform == 'win32': # ASan on Windows uses dbghelp.dll to symbolize in-process, which works # even in sandboxed processes. Nothing needs to be done here. @@ -371,7 +374,6 @@ class SymbolizationLoop(object): else: # Used by clients who may want to supply a different binary name. # E.g. in Chrome several binaries may share a single .dSYM. - self.binary_name_filter = binary_name_filter self.dsym_hint_producer = dsym_hint_producer self.system = os.uname()[0] if self.system not in ['Linux', 'Darwin', 'FreeBSD', 'NetBSD','SunOS']: @@ -455,8 +457,7 @@ class SymbolizationLoop(object): match = re.match(stack_trace_line_format, line) if not match: return [self.current_line] - if DEBUG: - print(line) + logging.debug(line) _, frameno_str, addr, binary, offset = match.groups() arch = "" # Arch can be embedded in the filename, e.g.: "libabc.dylib:x86_64h" @@ -472,52 +473,522 @@ class SymbolizationLoop(object): # Assume that frame #0 is the first frame of new stack trace. self.frame_no = 0 original_binary = binary - if self.binary_name_filter: - binary = self.binary_name_filter(binary) + binary = self.plugin_proxy.filter_binary_path(binary) + if binary is None: + # The binary filter has told us this binary can't be symbolized. + logging.debug('Skipping symbolication of binary "%s"', original_binary) + return [self.current_line] symbolized_line = self.symbolize_address(addr, binary, offset, arch) if not symbolized_line: if original_binary != binary: symbolized_line = self.symbolize_address(addr, original_binary, offset, arch) return self.get_symbolized_lines(symbolized_line) +class AsanSymbolizerPlugInProxy(object): + """ + Serves several purposes: + - Manages the lifetime of plugins (must be used a `with` statement). + - Provides interface for calling into plugins from within this script. + """ + def __init__(self): + self._plugins = [ ] + self._plugin_names = set() + + def _load_plugin_from_file_impl_py_gt_2(self, file_path, globals_space): + with open(file_path, 'r') as f: + exec(f.read(), globals_space, None) + + def load_plugin_from_file(self, file_path): + logging.info('Loading plugins from "{}"'.format(file_path)) + globals_space = dict(globals()) + # Provide function to register plugins + def register_plugin(plugin): + logging.info('Registering plugin %s', plugin.get_name()) + self.add_plugin(plugin) + globals_space['register_plugin'] = register_plugin + if sys.version_info.major < 3: + execfile(file_path, globals_space, None) + else: + # Indirection here is to avoid a bug in older Python 2 versions: + # `SyntaxError: unqualified exec is not allowed in function ...` + self._load_plugin_from_file_impl_py_gt_2(file_path, globals_space) + + def add_plugin(self, plugin): + assert isinstance(plugin, AsanSymbolizerPlugIn) + self._plugins.append(plugin) + self._plugin_names.add(plugin.get_name()) + plugin._receive_proxy(self) + + def remove_plugin(self, plugin): + assert isinstance(plugin, AsanSymbolizerPlugIn) + self._plugins.remove(plugin) + self._plugin_names.remove(plugin.get_name()) + logging.debug('Removing plugin %s', plugin.get_name()) + plugin.destroy() + + def has_plugin(self, name): + """ + Returns true iff the plugin name is currently + being managed by AsanSymbolizerPlugInProxy. + """ + return name in self._plugin_names + + def register_cmdline_args(self, parser): + plugins = list(self._plugins) + for plugin in plugins: + plugin.register_cmdline_args(parser) + + def process_cmdline_args(self, pargs): + # Use copy so we can remove items as we iterate. + plugins = list(self._plugins) + for plugin in plugins: + keep = plugin.process_cmdline_args(pargs) + assert isinstance(keep, bool) + if not keep: + self.remove_plugin(plugin) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for plugin in self._plugins: + plugin.destroy() + # Don't suppress raised exceptions + return False + + def _filter_single_value(self, function_name, input_value): + """ + Helper for filter style plugin functions. + """ + new_value = input_value + for plugin in self._plugins: + result = getattr(plugin, function_name)(new_value) + if result is None: + return None + new_value = result + return new_value + + def filter_binary_path(self, binary_path): + """ + Consult available plugins to filter the path to a binary + to make it suitable for symbolication. -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='ASan symbolization script', - epilog='Example of use:\n' - 'asan_symbolize.py -c "$HOME/opt/cross/bin/arm-linux-gnueabi-" ' - '-s "$HOME/SymbolFiles" < asan.log') - parser.add_argument('path_to_cut', nargs='*', - help='pattern to be cut from the result file path ') - parser.add_argument('-d','--demangle', action='store_true', - help='demangle function names') - parser.add_argument('-s', metavar='SYSROOT', + Returns `None` if symbolication should not be attempted for this + binary. + """ + return self._filter_single_value('filter_binary_path', binary_path) + + def filter_module_desc(self, module_desc): + """ + Consult available plugins to determine the module + description suitable for symbolication. + + Returns `None` if symbolication should not be attempted for this module. + """ + assert isinstance(module_desc, ModuleDesc) + return self._filter_single_value('filter_module_desc', module_desc) + +class AsanSymbolizerPlugIn(object): + """ + This is the interface the `asan_symbolize.py` code uses to talk + to plugins. + """ + @classmethod + def get_name(cls): + """ + Returns the name of the plugin. + """ + return cls.__name__ + + def _receive_proxy(self, proxy): + assert isinstance(proxy, AsanSymbolizerPlugInProxy) + self.proxy = proxy + + def register_cmdline_args(self, parser): + """ + Hook for registering command line arguments to be + consumed in `process_cmdline_args()`. + + `parser` - Instance of `argparse.ArgumentParser`. + """ + pass + + def process_cmdline_args(self, pargs): + """ + Hook for handling parsed arguments. Implementations + should not modify `pargs`. + + `pargs` - Instance of `argparse.Namespace` containing + parsed command line arguments. + + Return `True` if plug-in should be used, otherwise + return `False`. + """ + return True + + def destroy(self): + """ + Hook called when a plugin is about to be destroyed. + Implementations should free any allocated resources here. + """ + pass + + # Symbolization hooks + def filter_binary_path(self, binary_path): + """ + Given a binary path return a binary path suitable for symbolication. + + Implementations should return `None` if symbolication of this binary + should be skipped. + """ + return binary_path + + def filter_module_desc(self, module_desc): + """ + Given a ModuleDesc object (`module_desc`) return + a ModuleDesc suitable for symbolication. + + Implementations should return `None` if symbolication of this binary + should be skipped. + """ + return module_desc + +class ModuleDesc(object): + def __init__(self, name, arch, start_addr, end_addr, module_path, uuid): + self.name = name + self.arch = arch + self.start_addr = start_addr + self.end_addr = end_addr + # Module path from an ASan report. + self.module_path = module_path + # Module for performing symbolization, by default same as above. + self.module_path_for_symbolization = module_path + self.uuid = uuid + assert self.is_valid() + + def __str__(self): + assert self.is_valid() + return "{name} {arch} {start_addr:#016x}-{end_addr:#016x} {module_path} {uuid}".format( + name=self.name, + arch=self.arch, + start_addr=self.start_addr, + end_addr=self.end_addr, + module_path=self.module_path if self.module_path == self.module_path_for_symbolization else '{} ({})'.format(self.module_path_for_symbolization, self.module_path), + uuid=self.uuid + ) + + def is_valid(self): + if not isinstance(self.name, str): + return False + if not isinstance(self.arch, str): + return False + if not isinstance(self.start_addr, int): + return False + if self.start_addr < 0: + return False + if not isinstance(self.end_addr, int): + return False + if self.end_addr <= self.start_addr: + return False + if not isinstance(self.module_path, str): + return False + if not os.path.isabs(self.module_path): + return False + if not isinstance(self.module_path_for_symbolization, str): + return False + if not os.path.isabs(self.module_path_for_symbolization): + return False + if not isinstance(self.uuid, str): + return False + return True + +class GetUUIDFromBinaryException(Exception): + def __init__(self, msg): + super(GetUUIDFromBinaryException, self).__init__(msg) + +_get_uuid_from_binary_cache = dict() + +def get_uuid_from_binary(path_to_binary, arch=None): + cache_key = (path_to_binary, arch) + cached_value = _get_uuid_from_binary_cache.get(cache_key) + if cached_value: + return cached_value + if not os.path.exists(path_to_binary): + raise GetUUIDFromBinaryException('Binary "{}" does not exist'.format(path_to_binary)) + cmd = [ '/usr/bin/otool', '-l'] + if arch: + cmd.extend(['-arch', arch]) + cmd.append(path_to_binary) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + # Look for this output: + # cmd LC_UUID + # cmdsize 24 + # uuid 4CA778FE-5BF9-3C45-AE59-7DF01B2BE83F + if isinstance(output, str): + output_str = output + else: + assert isinstance(output, bytes) + output_str = output.decode() + assert isinstance(output_str, str) + lines = output_str.split('\n') + uuid = None + for index, line in enumerate(lines): + stripped_line = line.strip() + if not stripped_line.startswith('cmd LC_UUID'): + continue + uuid_line = lines[index+2].strip() + if not uuid_line.startswith('uuid'): + raise GetUUIDFromBinaryException('Malformed output: "{}"'.format(uuid_line)) + split_uuid_line = uuid_line.split() + uuid = split_uuid_line[1] + break + if uuid is None: + raise GetUUIDFromBinaryException('Failed to retrieve UUID') + else: + # Update cache + _get_uuid_from_binary_cache[cache_key] = uuid + return uuid + +class ModuleMap(object): + def __init__(self): + self._module_name_to_description_map = dict() + + def add_module(self, desc): + assert isinstance(desc, ModuleDesc) + assert desc.name not in self._module_name_to_description_map + self._module_name_to_description_map[desc.name] = desc + + def find_module_by_name(self, name): + return self._module_name_to_description_map.get(name, None) + + def __str__(self): + s = '{} modules:\n'.format(self.num_modules) + for module_desc in sorted(self._module_name_to_description_map.values(), key=lambda v: v.start_addr): + s += str(module_desc) + '\n' + return s + + @property + def num_modules(self): + return len(self._module_name_to_description_map) + + @property + def modules(self): + return set(self._module_name_to_description_map.values()) + + def get_module_path_for_symbolication(self, module_name, proxy): + module_desc = self.find_module_by_name(module_name) + if module_desc is None: + return None + # Allow a plug-in to change the module description to make it + # suitable for symbolication or avoid symbolication altogether. + module_desc = proxy.filter_module_desc(module_desc) + if module_desc is None: + return None + try: + uuid = get_uuid_from_binary(module_desc.module_path_for_symbolization, arch = module_desc.arch) + if uuid != module_desc.uuid: + logging.warning("Detected UUID mismatch {} != {}".format(uuid, module_desc.uuid)) + # UUIDs don't match. Tell client to not symbolize this. + return None + except GetUUIDFromBinaryException as e: + logging.error('Failed to binary from UUID: %s', str(e)) + return None + return module_desc.module_path_for_symbolization + + @staticmethod + def parse_from_file(module_map_path): + if not os.path.exists(module_map_path): + raise Exception('module map "{}" does not exist'.format(module_map_path)) + with open(module_map_path, 'r') as f: + mm = None + # E.g. + # 0x2db4000-0x102ddc000 /path/to (arm64) <0D6BBDE0-FF90-3680-899D-8E6F9528E04C> + hex_regex = lambda name: r'0x(?P<' + name + r'>[0-9a-f]+)' + module_path_regex = r'(?P<path>.+)' + arch_regex = r'\((?P<arch>.+)\)' + uuid_regex = r'<(?P<uuid>[0-9A-Z-]+)>' + line_regex = r'^{}-{}\s+{}\s+{}\s+{}'.format( + hex_regex('start_addr'), + hex_regex('end_addr'), + module_path_regex, + arch_regex, + uuid_regex + ) + matcher = re.compile(line_regex) + line_num = 0 + line = 'dummy' + while line != '': + line = f.readline() + line_num += 1 + if mm is None: + if line.startswith('Process module map:'): + mm = ModuleMap() + continue + if line.startswith('End of module map'): + break + m_obj = matcher.match(line) + if not m_obj: + raise Exception('Failed to parse line {} "{}"'.format(line_num, line)) + arch = m_obj.group('arch') + start_addr = int(m_obj.group('start_addr'), base=16) + end_addr = int(m_obj.group('end_addr'), base=16) + module_path = m_obj.group('path') + uuid = m_obj.group('uuid') + module_desc = ModuleDesc( + name=os.path.basename(module_path), + arch=arch, + start_addr=start_addr, + end_addr=end_addr, + module_path=module_path, + uuid=uuid + ) + mm.add_module(module_desc) + if mm is not None: + logging.debug('Loaded Module map from "{}":\n{}'.format( + f.name, + str(mm)) + ) + return mm + +class SysRootFilterPlugIn(AsanSymbolizerPlugIn): + """ + Simple plug-in to add sys root prefix to all binary paths + used for symbolication. + """ + def __init__(self): + self.sysroot_path = "" + + def register_cmdline_args(self, parser): + parser.add_argument('-s', dest='sys_root', metavar='SYSROOT', help='set path to sysroot for sanitized binaries') - parser.add_argument('-c', metavar='CROSS_COMPILE', - help='set prefix for binutils') - parser.add_argument('-l','--logfile', default=sys.stdin, - type=argparse.FileType('r'), - help='set log file name to parse, default is stdin') - parser.add_argument('--force-system-symbolizer', action='store_true', - help='don\'t use llvm-symbolizer') - args = parser.parse_args() - if args.path_to_cut: - fix_filename_patterns = args.path_to_cut - if args.demangle: - demangle = True - if args.s: - binary_name_filter = sysroot_path_filter - sysroot_path = args.s - if args.c: - binutils_prefix = args.c - if args.logfile: - logfile = args.logfile + + def process_cmdline_args(self, pargs): + if pargs.sys_root is None: + # Not being used so remove ourselves. + return False + self.sysroot_path = pargs.sys_root + return True + + def filter_binary_path(self, path): + return self.sysroot_path + path + +class ModuleMapPlugIn(AsanSymbolizerPlugIn): + def __init__(self): + self._module_map = None + def register_cmdline_args(self, parser): + parser.add_argument('--module-map', + help='Path to text file containing module map' + 'output. See print_module_map ASan option.') + def process_cmdline_args(self, pargs): + if not pargs.module_map: + return False + self._module_map = ModuleMap.parse_from_file(args.module_map) + if self._module_map is None: + msg = 'Failed to find module map' + logging.error(msg) + raise Exception(msg) + return True + def filter_binary_path(self, binary_path): + if os.path.isabs(binary_path): + # This is a binary path so transform into + # a module name + module_name = os.path.basename(binary_path) + else: + module_name = binary_path + return self._module_map.get_module_path_for_symbolication(module_name, self.proxy) + +def add_logging_args(parser): + parser.add_argument('--log-dest', + default=None, + help='Destination path for script logging (default stderr).', + ) + parser.add_argument('--log-level', + choices=['debug', 'info', 'warning', 'error', 'critical'], + default='info', + help='Log level for script (default: %(default)s).' + ) + +def setup_logging(): + # Set up a parser just for parsing the logging arguments. + # This is necessary because logging should be configured before we + # perform the main argument parsing. + parser = argparse.ArgumentParser(add_help=False) + add_logging_args(parser) + pargs, unparsed_args = parser.parse_known_args() + + log_level = getattr(logging, pargs.log_level.upper()) + if log_level == logging.DEBUG: + log_format = '%(levelname)s: [%(funcName)s() %(filename)s:%(lineno)d] %(message)s' else: - logfile = sys.stdin - if args.force_system_symbolizer: - force_system_symbolizer = True - if force_system_symbolizer: - assert(allow_system_symbolizer) - loop = SymbolizationLoop(binary_name_filter) - loop.process_logfile() + log_format = '%(levelname)s: %(message)s' + basic_config = { + 'level': log_level, + 'format': log_format + } + log_dest = pargs.log_dest + if log_dest: + basic_config['filename'] = log_dest + logging.basicConfig(**basic_config) + logging.debug('Logging level set to "{}" and directing output to "{}"'.format( + pargs.log_level, + 'stderr' if log_dest is None else log_dest) + ) + return unparsed_args + +def add_load_plugin_args(parser): + parser.add_argument('-p', '--plugins', + help='Load plug-in', nargs='+', default=[]) + +def setup_plugins(plugin_proxy, args): + parser = argparse.ArgumentParser(add_help=False) + add_load_plugin_args(parser) + pargs , unparsed_args = parser.parse_known_args() + for plugin_path in pargs.plugins: + plugin_proxy.load_plugin_from_file(plugin_path) + # Add built-in plugins. + plugin_proxy.add_plugin(ModuleMapPlugIn()) + plugin_proxy.add_plugin(SysRootFilterPlugIn()) + return unparsed_args + +if __name__ == '__main__': + remaining_args = setup_logging() + with AsanSymbolizerPlugInProxy() as plugin_proxy: + remaining_args = setup_plugins(plugin_proxy, remaining_args) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='ASan symbolization script', + epilog=__doc__) + parser.add_argument('path_to_cut', nargs='*', + help='pattern to be cut from the result file path ') + parser.add_argument('-d','--demangle', action='store_true', + help='demangle function names') + parser.add_argument('-c', metavar='CROSS_COMPILE', + help='set prefix for binutils') + parser.add_argument('-l','--logfile', default=sys.stdin, + type=argparse.FileType('r'), + help='set log file name to parse, default is stdin') + parser.add_argument('--force-system-symbolizer', action='store_true', + help='don\'t use llvm-symbolizer') + # Add logging arguments so that `--help` shows them. + add_logging_args(parser) + # Add load plugin arguments so that `--help` shows them. + add_load_plugin_args(parser) + plugin_proxy.register_cmdline_args(parser) + args = parser.parse_args(remaining_args) + plugin_proxy.process_cmdline_args(args) + if args.path_to_cut: + fix_filename_patterns = args.path_to_cut + if args.demangle: + demangle = True + if args.c: + binutils_prefix = args.c + if args.logfile: + logfile = args.logfile + else: + logfile = sys.stdin + if args.force_system_symbolizer: + force_system_symbolizer = True + if force_system_symbolizer: + assert(allow_system_symbolizer) + loop = SymbolizationLoop(plugin_proxy) + loop.process_logfile() diff --git a/lib/asan/tests/CMakeLists.txt b/lib/asan/tests/CMakeLists.txt index 9e640d1d8..d7116f7ff 100644 --- a/lib/asan/tests/CMakeLists.txt +++ b/lib/asan/tests/CMakeLists.txt @@ -74,10 +74,6 @@ set(ASAN_UNITTEST_INSTRUMENTED_CFLAGS -fsanitize=address "-fsanitize-blacklist=${ASAN_BLACKLIST_FILE}" ) -if(CAN_TARGET_x86_64 OR CAN_TARGET_i386) - list(APPEND ASAN_UNITTEST_INSTRUMENTED_CFLAGS -mllvm -asan-instrument-assembly) -endif() - if(NOT MSVC) list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS --driver-mode=g++) endif() @@ -142,7 +138,6 @@ set(ASAN_NOINST_TEST_SOURCES set(ASAN_INST_TEST_SOURCES ${COMPILER_RT_GTEST_SOURCE} - asan_asm_test.cc asan_globals_test.cc asan_interface_test.cc asan_internal_interface_test.cc diff --git a/lib/asan/tests/asan_asm_test.cc b/lib/asan/tests/asan_asm_test.cc deleted file mode 100644 index 91f8aacd6..000000000 --- a/lib/asan/tests/asan_asm_test.cc +++ /dev/null @@ -1,274 +0,0 @@ -//===-- asan_asm_test.cc --------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of AddressSanitizer, an address sanity checker. -// -//===----------------------------------------------------------------------===// -#include "asan_test_utils.h" - -#if defined(__linux__) && \ - (!defined(ASAN_SHADOW_SCALE) || ASAN_SHADOW_SCALE == 3) - -// Assembly instrumentation is broken on x86 Android (x86 + PIC + shared runtime -// library). See https://github.com/google/sanitizers/issues/353 -#if defined(__x86_64__) || \ - (defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__)) - -#include <emmintrin.h> - -namespace { - -template<typename T> void asm_write(T *ptr, T val); -template<typename T> T asm_read(T *ptr); -template<typename T> void asm_rep_movs(T *dst, T *src, size_t n); - -} // End of anonymous namespace - -#endif // defined(__x86_64__) || (defined(__i386__) && defined(__SSE2__)) - -#if defined(__x86_64__) - -namespace { - -#define DECLARE_ASM_WRITE(Type, Size, Mov, Reg) \ -template<> void asm_write<Type>(Type *ptr, Type val) { \ - __asm__( \ - Mov " %[val], (%[ptr]) \n\t" \ - : \ - : [ptr] "r" (ptr), [val] Reg (val) \ - : "memory" \ - ); \ -} - -#define DECLARE_ASM_READ(Type, Size, Mov, Reg) \ -template<> Type asm_read<Type>(Type *ptr) { \ - Type res; \ - __asm__( \ - Mov " (%[ptr]), %[res] \n\t" \ - : [res] Reg (res) \ - : [ptr] "r" (ptr) \ - : "memory" \ - ); \ - return res; \ -} - -#define DECLARE_ASM_REP_MOVS(Type, Movs) \ - template <> \ - void asm_rep_movs<Type>(Type * dst, Type * src, size_t size) { \ - __asm__("rep " Movs " \n\t" \ - : "+D"(dst), "+S"(src), "+c"(size) \ - : \ - : "memory"); \ - } - -DECLARE_ASM_WRITE(U8, "8", "movq", "r"); -DECLARE_ASM_READ(U8, "8", "movq", "=r"); -DECLARE_ASM_REP_MOVS(U8, "movsq"); - -} // End of anonymous namespace - -#endif // defined(__x86_64__) - -#if defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__) - -namespace { - -#define DECLARE_ASM_WRITE(Type, Size, Mov, Reg) \ -template<> void asm_write<Type>(Type *ptr, Type val) { \ - __asm__( \ - Mov " %[val], (%[ptr]) \n\t" \ - : \ - : [ptr] "r" (ptr), [val] Reg (val) \ - : "memory" \ - ); \ -} - -#define DECLARE_ASM_READ(Type, Size, Mov, Reg) \ -template<> Type asm_read<Type>(Type *ptr) { \ - Type res; \ - __asm__( \ - Mov " (%[ptr]), %[res] \n\t" \ - : [res] Reg (res) \ - : [ptr] "r" (ptr) \ - : "memory" \ - ); \ - return res; \ -} - -#define DECLARE_ASM_REP_MOVS(Type, Movs) \ - template <> \ - void asm_rep_movs<Type>(Type * dst, Type * src, size_t size) { \ - __asm__("rep " Movs " \n\t" \ - : "+D"(dst), "+S"(src), "+c"(size) \ - : \ - : "memory"); \ - } - -} // End of anonymous namespace - -#endif // defined(__i386__) && defined(__SSE2__) - -#if defined(__x86_64__) || \ - (defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__)) - -namespace { - -DECLARE_ASM_WRITE(U1, "1", "movb", "r"); -DECLARE_ASM_WRITE(U2, "2", "movw", "r"); -DECLARE_ASM_WRITE(U4, "4", "movl", "r"); -DECLARE_ASM_WRITE(__m128i, "16", "movaps", "x"); - -DECLARE_ASM_READ(U1, "1", "movb", "=r"); -DECLARE_ASM_READ(U2, "2", "movw", "=r"); -DECLARE_ASM_READ(U4, "4", "movl", "=r"); -DECLARE_ASM_READ(__m128i, "16", "movaps", "=x"); - -DECLARE_ASM_REP_MOVS(U1, "movsb"); -DECLARE_ASM_REP_MOVS(U2, "movsw"); -DECLARE_ASM_REP_MOVS(U4, "movsl"); - -template<typename T> void TestAsmWrite(const char *DeathPattern) { - T *buf = new T; - EXPECT_DEATH(asm_write(&buf[1], static_cast<T>(0)), DeathPattern); - T var = 0x12; - asm_write(&var, static_cast<T>(0x21)); - ASSERT_EQ(static_cast<T>(0x21), var); - delete buf; -} - -template<> void TestAsmWrite<__m128i>(const char *DeathPattern) { - char *buf = new char[16]; - char *p = buf + 16; - if (((uintptr_t) p % 16) != 0) - p = buf + 8; - assert(((uintptr_t) p % 16) == 0); - __m128i val = _mm_set1_epi16(0x1234); - EXPECT_DEATH(asm_write<__m128i>((__m128i*) p, val), DeathPattern); - __m128i var = _mm_set1_epi16(0x4321); - asm_write(&var, val); - ASSERT_EQ(0x1234, _mm_extract_epi16(var, 0)); - delete [] buf; -} - -template<typename T> void TestAsmRead(const char *DeathPattern) { - T *buf = new T; - EXPECT_DEATH(asm_read(&buf[1]), DeathPattern); - T var = 0x12; - ASSERT_EQ(static_cast<T>(0x12), asm_read(&var)); - delete buf; -} - -template<> void TestAsmRead<__m128i>(const char *DeathPattern) { - char *buf = new char[16]; - char *p = buf + 16; - if (((uintptr_t) p % 16) != 0) - p = buf + 8; - assert(((uintptr_t) p % 16) == 0); - EXPECT_DEATH(asm_read<__m128i>((__m128i*) p), DeathPattern); - __m128i val = _mm_set1_epi16(0x1234); - ASSERT_EQ(0x1234, _mm_extract_epi16(asm_read(&val), 0)); - delete [] buf; -} - -U4 AsmLoad(U4 *a) { - U4 r; - __asm__("movl (%[a]), %[r] \n\t" : [r] "=r" (r) : [a] "r" (a) : "memory"); - return r; -} - -void AsmStore(U4 r, U4 *a) { - __asm__("movl %[r], (%[a]) \n\t" : : [a] "r" (a), [r] "r" (r) : "memory"); -} - -template <typename T> -void TestAsmRepMovs(const char *DeathPatternRead, - const char *DeathPatternWrite) { - T src_good[4] = { 0x0, 0x1, 0x2, 0x3 }; - T dst_good[4] = {}; - asm_rep_movs(dst_good, src_good, 4); - ASSERT_EQ(static_cast<T>(0x0), dst_good[0]); - ASSERT_EQ(static_cast<T>(0x1), dst_good[1]); - ASSERT_EQ(static_cast<T>(0x2), dst_good[2]); - ASSERT_EQ(static_cast<T>(0x3), dst_good[3]); - - T dst_bad[3]; - EXPECT_DEATH(asm_rep_movs(dst_bad, src_good, 4), DeathPatternWrite); - - T src_bad[3] = { 0x0, 0x1, 0x2 }; - EXPECT_DEATH(asm_rep_movs(dst_good, src_bad, 4), DeathPatternRead); - - T* dp = dst_bad + 4; - T* sp = src_bad + 4; - asm_rep_movs(dp, sp, 0); -} - -} // End of anonymous namespace - -TEST(AddressSanitizer, asm_load_store) { - U4* buf = new U4[2]; - EXPECT_DEATH(AsmLoad(&buf[3]), "READ of size 4"); - EXPECT_DEATH(AsmStore(0x1234, &buf[3]), "WRITE of size 4"); - delete [] buf; -} - -TEST(AddressSanitizer, asm_rw) { - TestAsmWrite<U1>("WRITE of size 1"); - TestAsmWrite<U2>("WRITE of size 2"); - TestAsmWrite<U4>("WRITE of size 4"); -#if defined(__x86_64__) - TestAsmWrite<U8>("WRITE of size 8"); -#endif // defined(__x86_64__) - TestAsmWrite<__m128i>("WRITE of size 16"); - - TestAsmRead<U1>("READ of size 1"); - TestAsmRead<U2>("READ of size 2"); - TestAsmRead<U4>("READ of size 4"); -#if defined(__x86_64__) - TestAsmRead<U8>("READ of size 8"); -#endif // defined(__x86_64__) - TestAsmRead<__m128i>("READ of size 16"); -} - -TEST(AddressSanitizer, asm_flags) { - long magic = 0x1234; - long r = 0x0; - -#if defined(__x86_64__) && !defined(__ILP32__) - __asm__("xorq %%rax, %%rax \n\t" - "movq (%[p]), %%rax \n\t" - "sete %%al \n\t" - "movzbq %%al, %[r] \n\t" - : [r] "=r"(r) - : [p] "r"(&magic) - : "rax", "memory"); -#else - __asm__("xorl %%eax, %%eax \n\t" - "movl (%[p]), %%eax \n\t" - "sete %%al \n\t" - "movzbl %%al, %[r] \n\t" - : [r] "=r"(r) - : [p] "r"(&magic) - : "eax", "memory"); -#endif // defined(__x86_64__) && !defined(__ILP32__) - - ASSERT_EQ(0x1, r); -} - -TEST(AddressSanitizer, asm_rep_movs) { - TestAsmRepMovs<U1>("READ of size 1", "WRITE of size 1"); - TestAsmRepMovs<U2>("READ of size 2", "WRITE of size 2"); - TestAsmRepMovs<U4>("READ of size 4", "WRITE of size 4"); -#if defined(__x86_64__) - TestAsmRepMovs<U8>("READ of size 8", "WRITE of size 8"); -#endif // defined(__x86_64__) -} - -#endif // defined(__x86_64__) || (defined(__i386__) && defined(__SSE2__)) - -#endif // defined(__linux__) diff --git a/lib/asan/tests/asan_benchmarks_test.cc b/lib/asan/tests/asan_benchmarks_test.cc index fc522de47..9a0ed72d4 100644 --- a/lib/asan/tests/asan_benchmarks_test.cc +++ b/lib/asan/tests/asan_benchmarks_test.cc @@ -1,9 +1,8 @@ //===-- asan_benchmarks_test.cc ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_fake_stack_test.cc b/lib/asan/tests/asan_fake_stack_test.cc index 516142f0c..13beea8f2 100644 --- a/lib/asan/tests/asan_fake_stack_test.cc +++ b/lib/asan/tests/asan_fake_stack_test.cc @@ -1,9 +1,8 @@ //===-- asan_fake_stack_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_globals_test.cc b/lib/asan/tests/asan_globals_test.cc index 5042ef073..b852b6b5e 100644 --- a/lib/asan/tests/asan_globals_test.cc +++ b/lib/asan/tests/asan_globals_test.cc @@ -1,9 +1,8 @@ //===-- asan_globals_test.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_interface_test.cc b/lib/asan/tests/asan_interface_test.cc index b5c8303cb..8cbe469bf 100644 --- a/lib/asan/tests/asan_interface_test.cc +++ b/lib/asan/tests/asan_interface_test.cc @@ -1,9 +1,8 @@ //===-- asan_interface_test.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_internal_interface_test.cc b/lib/asan/tests/asan_internal_interface_test.cc index e247bb429..2d3b9c1f5 100644 --- a/lib/asan/tests/asan_internal_interface_test.cc +++ b/lib/asan/tests/asan_internal_interface_test.cc @@ -1,9 +1,8 @@ //===-- asan_internal_interface_test.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_mac_test.cc b/lib/asan/tests/asan_mac_test.cc index dfa6d7596..14e105252 100644 --- a/lib/asan/tests/asan_mac_test.cc +++ b/lib/asan/tests/asan_mac_test.cc @@ -1,9 +1,8 @@ //===-- asan_test_mac.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_mem_test.cc b/lib/asan/tests/asan_mem_test.cc index c32088684..1339c1271 100644 --- a/lib/asan/tests/asan_mem_test.cc +++ b/lib/asan/tests/asan_mem_test.cc @@ -1,16 +1,19 @@ //===-- asan_mem_test.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of AddressSanitizer, an address sanity checker. // //===----------------------------------------------------------------------===// +#include <string.h> #include "asan_test_utils.h" +#if defined(_GNU_SOURCE) +#include <strings.h> // for bcmp +#endif #include <vector> template<typename T> @@ -206,37 +209,43 @@ TEST(AddressSanitizer, MemMoveOOBTest) { MemTransferOOBTestTemplate<int, MemMoveWrapper>(1024); } - -TEST(AddressSanitizer, MemCmpOOBTest) { +template <int (*cmpfn)(const void *, const void *, size_t)> +void CmpOOBTestCommon() { size_t size = Ident(100); char *s1 = MallocAndMemsetString(size); char *s2 = MallocAndMemsetString(size); - // Normal memcmp calls. - Ident(memcmp(s1, s2, size)); - Ident(memcmp(s1 + size - 1, s2 + size - 1, 1)); - Ident(memcmp(s1 - 1, s2 - 1, 0)); + // Normal cmpfn calls. + Ident(cmpfn(s1, s2, size)); + Ident(cmpfn(s1 + size - 1, s2 + size - 1, 1)); + Ident(cmpfn(s1 - 1, s2 - 1, 0)); // One of arguments points to not allocated memory. - EXPECT_DEATH(Ident(memcmp)(s1 - 1, s2, 1), LeftOOBReadMessage(1)); - EXPECT_DEATH(Ident(memcmp)(s1, s2 - 1, 1), LeftOOBReadMessage(1)); - EXPECT_DEATH(Ident(memcmp)(s1 + size, s2, 1), RightOOBReadMessage(0)); - EXPECT_DEATH(Ident(memcmp)(s1, s2 + size, 1), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1 - 1, s2, 1), LeftOOBReadMessage(1)); + EXPECT_DEATH(Ident(cmpfn)(s1, s2 - 1, 1), LeftOOBReadMessage(1)); + EXPECT_DEATH(Ident(cmpfn)(s1 + size, s2, 1), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1, s2 + size, 1), RightOOBReadMessage(0)); // Hit unallocated memory and die. - EXPECT_DEATH(Ident(memcmp)(s1 + 1, s2 + 1, size), RightOOBReadMessage(0)); - EXPECT_DEATH(Ident(memcmp)(s1 + size - 1, s2, 2), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1 + 1, s2 + 1, size), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1 + size - 1, s2, 2), RightOOBReadMessage(0)); // Zero bytes are not terminators and don't prevent from OOB. s1[size - 1] = '\0'; s2[size - 1] = '\0'; - EXPECT_DEATH(Ident(memcmp)(s1, s2, size + 1), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1, s2, size + 1), RightOOBReadMessage(0)); // Even if the buffers differ in the first byte, we still assume that - // memcmp may access the whole buffer and thus reporting the overflow here: + // cmpfn may access the whole buffer and thus reporting the overflow here: s1[0] = 1; s2[0] = 123; - EXPECT_DEATH(Ident(memcmp)(s1, s2, size + 1), RightOOBReadMessage(0)); + EXPECT_DEATH(Ident(cmpfn)(s1, s2, size + 1), RightOOBReadMessage(0)); free(s1); free(s2); } +TEST(AddressSanitizer, MemCmpOOBTest) { CmpOOBTestCommon<memcmp>(); } - +TEST(AddressSanitizer, BCmpOOBTest) { +#if (defined(__linux__) && !defined(__ANDROID__) && defined(_GNU_SOURCE)) || \ + defined(__NetBSD__) || defined(__FreeBSD__) || defined(__OpenBSD__) + CmpOOBTestCommon<bcmp>(); +#endif +} diff --git a/lib/asan/tests/asan_noinst_test.cc b/lib/asan/tests/asan_noinst_test.cc index 3e366842c..8460abf97 100644 --- a/lib/asan/tests/asan_noinst_test.cc +++ b/lib/asan/tests/asan_noinst_test.cc @@ -1,9 +1,8 @@ //===-- asan_noinst_test.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_oob_test.cc b/lib/asan/tests/asan_oob_test.cc index 0c6bea285..f023ca24b 100644 --- a/lib/asan/tests/asan_oob_test.cc +++ b/lib/asan/tests/asan_oob_test.cc @@ -1,9 +1,8 @@ //===-- asan_oob_test.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_str_test.cc b/lib/asan/tests/asan_str_test.cc index 5cf4e05c8..8ed451408 100644 --- a/lib/asan/tests/asan_str_test.cc +++ b/lib/asan/tests/asan_str_test.cc @@ -1,9 +1,8 @@ //=-- asan_str_test.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_test.cc b/lib/asan/tests/asan_test.cc index 464e99f61..2795b71cd 100644 --- a/lib/asan/tests/asan_test.cc +++ b/lib/asan/tests/asan_test.cc @@ -1,9 +1,8 @@ //===-- asan_test.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_test_config.h b/lib/asan/tests/asan_test_config.h index 8493f41ef..a0fadf8af 100644 --- a/lib/asan/tests/asan_test_config.h +++ b/lib/asan/tests/asan_test_config.h @@ -1,9 +1,8 @@ //===-- asan_test_config.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_test_main.cc b/lib/asan/tests/asan_test_main.cc index 0c1b93c7f..1f94ec287 100644 --- a/lib/asan/tests/asan_test_main.cc +++ b/lib/asan/tests/asan_test_main.cc @@ -1,9 +1,8 @@ //===-- asan_test_main.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/tests/asan_test_utils.h b/lib/asan/tests/asan_test_utils.h index d7b6f82e2..2ab44855a 100644 --- a/lib/asan/tests/asan_test_utils.h +++ b/lib/asan/tests/asan_test_utils.h @@ -1,9 +1,8 @@ //===-- asan_test_utils.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/CMakeLists.txt b/lib/builtins/CMakeLists.txt index 77947417b..d0bd07b63 100644 --- a/lib/builtins/CMakeLists.txt +++ b/lib/builtins/CMakeLists.txt @@ -150,7 +150,8 @@ set(GENERIC_SOURCES udivti3.c umoddi3.c umodsi3.c - umodti3.c) + umodti3.c +) set(GENERIC_TF_SOURCES comparetf2.c @@ -170,7 +171,8 @@ set(GENERIC_TF_SOURCES floatuntitf.c multc3.c trunctfdf2.c - trunctfsf2.c) + trunctfsf2.c +) option(COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN "Skip the atomic builtin (these should normally be provided by a shared library)" @@ -179,15 +181,17 @@ option(COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN if(NOT FUCHSIA AND NOT COMPILER_RT_BAREMETAL_BUILD) set(GENERIC_SOURCES ${GENERIC_SOURCES} - emutls.c + emutls.c enable_execute_stack.c - eprintf.c) + eprintf.c + ) endif() if(COMPILER_RT_HAS_ATOMIC_KEYWORD AND NOT COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN) set(GENERIC_SOURCES ${GENERIC_SOURCES} - atomic.c) + atomic.c + ) endif() if(APPLE) @@ -198,19 +202,22 @@ if(APPLE) atomic_flag_test_and_set.c atomic_flag_test_and_set_explicit.c atomic_signal_fence.c - atomic_thread_fence.c) + atomic_thread_fence.c + ) endif() if (HAVE_UNWIND_H) set(GENERIC_SOURCES - ${GENERIC_SOURCES} - gcc_personality_v0.c) + ${GENERIC_SOURCES} + gcc_personality_v0.c + ) endif () if (NOT FUCHSIA) set(GENERIC_SOURCES ${GENERIC_SOURCES} - clear_cache.c) + clear_cache.c + ) endif() # These sources work on all x86 variants, but only x86 variants. @@ -232,54 +239,60 @@ set(x86_ARCH_SOURCES if (NOT MSVC) set(x86_64_SOURCES - x86_64/floatdidf.c - x86_64/floatdisf.c - x86_64/floatdixf.c - x86_64/floatundidf.S - x86_64/floatundisf.S - x86_64/floatundixf.S) + ${GENERIC_TF_SOURCES} + x86_64/floatdidf.c + x86_64/floatdisf.c + x86_64/floatdixf.c + x86_64/floatundidf.S + x86_64/floatundisf.S + x86_64/floatundixf.S + ) filter_builtin_sources(x86_64_SOURCES EXCLUDE x86_64_SOURCES "${x86_64_SOURCES};${GENERIC_SOURCES}") set(x86_64h_SOURCES ${x86_64_SOURCES}) if (WIN32) set(x86_64_SOURCES - ${x86_64_SOURCES} - x86_64/chkstk.S - x86_64/chkstk2.S) + ${x86_64_SOURCES} + x86_64/chkstk.S + x86_64/chkstk2.S + ) endif() set(i386_SOURCES - i386/ashldi3.S - i386/ashrdi3.S - i386/divdi3.S - i386/floatdidf.S - i386/floatdisf.S - i386/floatdixf.S - i386/floatundidf.S - i386/floatundisf.S - i386/floatundixf.S - i386/lshrdi3.S - i386/moddi3.S - i386/muldi3.S - i386/udivdi3.S - i386/umoddi3.S) + i386/ashldi3.S + i386/ashrdi3.S + i386/divdi3.S + i386/floatdidf.S + i386/floatdisf.S + i386/floatdixf.S + i386/floatundidf.S + i386/floatundisf.S + i386/floatundixf.S + i386/lshrdi3.S + i386/moddi3.S + i386/muldi3.S + i386/udivdi3.S + i386/umoddi3.S + ) filter_builtin_sources(i386_SOURCES EXCLUDE i386_SOURCES "${i386_SOURCES};${GENERIC_SOURCES}") if (WIN32) set(i386_SOURCES - ${i386_SOURCES} - i386/chkstk.S - i386/chkstk2.S) + ${i386_SOURCES} + i386/chkstk.S + i386/chkstk2.S + ) endif() else () # MSVC # Use C versions of functions when building on MSVC # MSVC's assembler takes Intel syntax, not AT&T syntax. # Also use only MSVC compilable builtin implementations. set(x86_64_SOURCES - x86_64/floatdidf.c - x86_64/floatdisf.c - x86_64/floatdixf.c - ${GENERIC_SOURCES}) + x86_64/floatdidf.c + x86_64/floatdisf.c + x86_64/floatdixf.c + ${GENERIC_SOURCES} + ) set(x86_64h_SOURCES ${x86_64_SOURCES}) set(i386_SOURCES ${GENERIC_SOURCES}) endif () # if (NOT MSVC) @@ -320,7 +333,8 @@ set(arm_SOURCES arm/sync_fetch_and_xor_8.S arm/udivmodsi4.S arm/udivsi3.S - arm/umodsi3.S) + arm/umodsi3.S +) filter_builtin_sources(arm_SOURCES EXCLUDE arm_SOURCES "${arm_SOURCES};${GENERIC_SOURCES}") set(thumb1_SOURCES @@ -328,7 +342,8 @@ set(thumb1_SOURCES arm/udivsi3.S arm/comparesf2.S arm/addsf3.S - ${GENERIC_SOURCES}) + ${GENERIC_SOURCES} +) set(arm_EABI_SOURCES arm/aeabi_cdcmp.S @@ -347,16 +362,19 @@ set(arm_EABI_SOURCES arm/aeabi_memmove.S arm/aeabi_memset.S arm/aeabi_uidivmod.S - arm/aeabi_uldivmod.S) + arm/aeabi_uldivmod.S +) set(arm_Thumb1_JT_SOURCES arm/switch16.S arm/switch32.S arm/switch8.S - arm/switchu8.S) + arm/switchu8.S +) set(arm_Thumb1_SjLj_EH_SOURCES arm/restore_vfp_d8_d15_regs.S - arm/save_vfp_d8_d15_regs.S) + arm/save_vfp_d8_d15_regs.S +) set(arm_Thumb1_VFPv2_SOURCES arm/adddf3vfp.S arm/addsf3vfp.S @@ -391,62 +409,70 @@ set(arm_Thumb1_VFPv2_SOURCES arm/subsf3vfp.S arm/truncdfsf2vfp.S arm/unorddf2vfp.S - arm/unordsf2vfp.S) + arm/unordsf2vfp.S +) set(arm_Thumb1_icache_SOURCES - arm/sync_synchronize.S) + arm/sync_synchronize.S +) set(arm_Thumb1_SOURCES ${arm_Thumb1_JT_SOURCES} ${arm_Thumb1_SjLj_EH_SOURCES} ${arm_Thumb1_VFPv2_SOURCES} - ${arm_Thumb1_icache_SOURCES}) + ${arm_Thumb1_icache_SOURCES} +) if(MINGW) set(arm_SOURCES - arm/aeabi_idivmod.S - arm/aeabi_ldivmod.S - arm/aeabi_uidivmod.S - arm/aeabi_uldivmod.S - arm/chkstk.S - divmoddi4.c - divmodsi4.c - divdi3.c - divsi3.c - fixdfdi.c - fixsfdi.c - fixunsdfdi.c - fixunssfdi.c - floatdidf.c - floatdisf.c - floatundidf.c - floatundisf.c - mingw_fixfloat.c - moddi3.c - udivmoddi4.c - udivmodsi4.c - udivsi3.c - umoddi3.c - emutls.c) + arm/aeabi_idivmod.S + arm/aeabi_ldivmod.S + arm/aeabi_uidivmod.S + arm/aeabi_uldivmod.S + arm/chkstk.S + divmoddi4.c + divmodsi4.c + divdi3.c + divsi3.c + fixdfdi.c + fixsfdi.c + fixunsdfdi.c + fixunssfdi.c + floatdidf.c + floatdisf.c + floatundidf.c + floatundisf.c + mingw_fixfloat.c + moddi3.c + udivmoddi4.c + udivmodsi4.c + udivsi3.c + umoddi3.c + emutls.c + ) filter_builtin_sources(arm_SOURCES EXCLUDE arm_SOURCES "${arm_SOURCES};${GENERIC_SOURCES}") elseif(NOT WIN32) # TODO the EABI sources should only be added to EABI targets set(arm_SOURCES ${arm_SOURCES} ${arm_EABI_SOURCES} - ${arm_Thumb1_SOURCES}) + ${arm_Thumb1_SOURCES} + ) set(thumb1_SOURCES ${thumb1_SOURCES} - ${arm_EABI_SOURCES}) + ${arm_EABI_SOURCES} + ) endif() set(aarch64_SOURCES ${GENERIC_TF_SOURCES} - ${GENERIC_SOURCES}) + ${GENERIC_SOURCES} +) if (MINGW) set(aarch64_SOURCES - ${aarch64_SOURCES} - aarch64/chkstk.S) + ${aarch64_SOURCES} + aarch64/chkstk.S + ) endif() set(armhf_SOURCES ${arm_SOURCES}) @@ -492,7 +518,8 @@ set(hexagon_SOURCES hexagon/udivmodsi4.S hexagon/udivsi3.S hexagon/umoddi3.S - hexagon/umodsi3.S) + hexagon/umodsi3.S +) set(mips_SOURCES ${GENERIC_SOURCES}) @@ -515,21 +542,25 @@ set(powerpc64_SOURCES ppc/gcc_qmul.c ppc/gcc_qsub.c ppc/multc3.c - ${GENERIC_SOURCES}) + ${GENERIC_SOURCES} +) set(powerpc64le_SOURCES ${powerpc64_SOURCES}) set(riscv_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES}) set(riscv32_SOURCES riscv/mulsi3.S - ${riscv_SOURCES}) + ${riscv_SOURCES} +) set(riscv64_SOURCES ${riscv_SOURCES}) set(wasm32_SOURCES ${GENERIC_TF_SOURCES} - ${GENERIC_SOURCES}) + ${GENERIC_SOURCES} +) set(wasm64_SOURCES ${GENERIC_TF_SOURCES} - ${GENERIC_SOURCES}) + ${GENERIC_SOURCES} +) add_custom_target(builtins) set_target_properties(builtins PROPERTIES FOLDER "Compiler-RT Misc") @@ -548,7 +579,9 @@ else () if(COMPILER_RT_STANDALONE_BUILD) append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC BUILTIN_CFLAGS) append_list_if(COMPILER_RT_HAS_FNO_BUILTIN_FLAG -fno-builtin BUILTIN_CFLAGS) - append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG -fvisibility=hidden BUILTIN_CFLAGS) + if(NOT ANDROID) + append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG -fvisibility=hidden BUILTIN_CFLAGS) + endif() if(NOT COMPILER_RT_DEBUG) append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fomit-frame-pointer BUILTIN_CFLAGS) endif() @@ -556,7 +589,9 @@ else () set(BUILTIN_DEFS "") - append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG VISIBILITY_HIDDEN BUILTIN_DEFS) + if(NOT ANDROID) + append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG VISIBILITY_HIDDEN BUILTIN_DEFS) + endif() foreach (arch ${BUILTIN_SUPPORTED_ARCH}) if (CAN_TARGET_${arch}) diff --git a/lib/builtins/aarch64/chkstk.S b/lib/builtins/aarch64/chkstk.S index 89ec90b08..01f90366f 100644 --- a/lib/builtins/aarch64/chkstk.S +++ b/lib/builtins/aarch64/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/absvdi2.c b/lib/builtins/absvdi2.c index 682c2355d..b9566cd87 100644 --- a/lib/builtins/absvdi2.c +++ b/lib/builtins/absvdi2.c @@ -1,29 +1,25 @@ -/*===-- absvdi2.c - Implement __absvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __absvdi2 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- absvdi2.c - Implement __absvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI di_int -__absvdi2(di_int a) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - if (a == ((di_int)1 << (N-1))) - compilerrt_abort(); - const di_int t = a >> (N - 1); - return (a ^ t) - t; +COMPILER_RT_ABI di_int __absvdi2(di_int a) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + if (a == ((di_int)1 << (N - 1))) + compilerrt_abort(); + const di_int t = a >> (N - 1); + return (a ^ t) - t; } diff --git a/lib/builtins/absvsi2.c b/lib/builtins/absvsi2.c index 4812af815..44ada169e 100644 --- a/lib/builtins/absvsi2.c +++ b/lib/builtins/absvsi2.c @@ -1,29 +1,25 @@ -/* ===-- absvsi2.c - Implement __absvsi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __absvsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- absvsi2.c - Implement __absvsi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI si_int -__absvsi2(si_int a) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - if (a == (1 << (N-1))) - compilerrt_abort(); - const si_int t = a >> (N - 1); - return (a ^ t) - t; +COMPILER_RT_ABI si_int __absvsi2(si_int a) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + if (a == (1 << (N - 1))) + compilerrt_abort(); + const si_int t = a >> (N - 1); + return (a ^ t) - t; } diff --git a/lib/builtins/absvti2.c b/lib/builtins/absvti2.c index 7927770c9..491d99d7c 100644 --- a/lib/builtins/absvti2.c +++ b/lib/builtins/absvti2.c @@ -1,34 +1,29 @@ -/* ===-- absvti2.c - Implement __absvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __absvti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- absvti2.c - Implement __absvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI ti_int -__absvti2(ti_int a) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - if (a == ((ti_int)1 << (N-1))) - compilerrt_abort(); - const ti_int s = a >> (N - 1); - return (a ^ s) - s; +COMPILER_RT_ABI ti_int __absvti2(ti_int a) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + if (a == ((ti_int)1 << (N - 1))) + compilerrt_abort(); + const ti_int s = a >> (N - 1); + return (a ^ s) - s; } -#endif /* CRT_HAS_128BIT */ - +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/adddf3.c b/lib/builtins/adddf3.c index 9a3901312..f2727fafc 100644 --- a/lib/builtins/adddf3.c +++ b/lib/builtins/adddf3.c @@ -1,9 +1,8 @@ //===-- lib/adddf3.c - Double-precision addition ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define DOUBLE_PRECISION #include "fp_add_impl.inc" -COMPILER_RT_ABI double __adddf3(double a, double b){ - return __addXf3__(a, b); -} +COMPILER_RT_ABI double __adddf3(double a, double b) { return __addXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_dadd(double a, double b) { - return __adddf3(a, b); -} +AEABI_RTABI double __aeabi_dadd(double a, double b) { return __adddf3(a, b); } #else -AEABI_RTABI double __aeabi_dadd(double a, double b) COMPILER_RT_ALIAS(__adddf3); +COMPILER_RT_ALIAS(__adddf3, __aeabi_dadd) #endif #endif diff --git a/lib/builtins/addsf3.c b/lib/builtins/addsf3.c index c5c1a41c3..8fe8622aa 100644 --- a/lib/builtins/addsf3.c +++ b/lib/builtins/addsf3.c @@ -1,9 +1,8 @@ //===-- lib/addsf3.c - Single-precision addition ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define SINGLE_PRECISION #include "fp_add_impl.inc" -COMPILER_RT_ABI float __addsf3(float a, float b) { - return __addXf3__(a, b); -} +COMPILER_RT_ABI float __addsf3(float a, float b) { return __addXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_fadd(float a, float b) { - return __addsf3(a, b); -} +AEABI_RTABI float __aeabi_fadd(float a, float b) { return __addsf3(a, b); } #else -AEABI_RTABI float __aeabi_fadd(float a, float b) COMPILER_RT_ALIAS(__addsf3); +COMPILER_RT_ALIAS(__addsf3, __aeabi_fadd) #endif #endif diff --git a/lib/builtins/addtf3.c b/lib/builtins/addtf3.c index e4bbe0227..570472a14 100644 --- a/lib/builtins/addtf3.c +++ b/lib/builtins/addtf3.c @@ -1,9 +1,8 @@ //===-- lib/addtf3.c - Quad-precision addition --------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) #include "fp_add_impl.inc" -COMPILER_RT_ABI long double __addtf3(long double a, long double b){ - return __addXf3__(a, b); +COMPILER_RT_ABI long double __addtf3(long double a, long double b) { + return __addXf3__(a, b); } #endif diff --git a/lib/builtins/addvdi3.c b/lib/builtins/addvdi3.c index 0da389456..28661fda8 100644 --- a/lib/builtins/addvdi3.c +++ b/lib/builtins/addvdi3.c @@ -1,36 +1,29 @@ -/* ===-- addvdi3.c - Implement __addvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvdi3.c - Implement __addvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI di_int -__addvdi3(di_int a, di_int b) -{ - di_int s = (du_int) a + (du_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI di_int __addvdi3(di_int a, di_int b) { + di_int s = (du_int)a + (du_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/addvsi3.c b/lib/builtins/addvsi3.c index 94ca726f4..404002375 100644 --- a/lib/builtins/addvsi3.c +++ b/lib/builtins/addvsi3.c @@ -1,36 +1,29 @@ -/* ===-- addvsi3.c - Implement __addvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvsi3.c - Implement __addvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI si_int -__addvsi3(si_int a, si_int b) -{ - si_int s = (su_int) a + (su_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI si_int __addvsi3(si_int a, si_int b) { + si_int s = (su_int)a + (su_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/addvti3.c b/lib/builtins/addvti3.c index c224de60a..aa709875d 100644 --- a/lib/builtins/addvti3.c +++ b/lib/builtins/addvti3.c @@ -1,40 +1,33 @@ -/* ===-- addvti3.c - Implement __addvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvti3.c - Implement __addvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI ti_int -__addvti3(ti_int a, ti_int b) -{ - ti_int s = (tu_int) a + (tu_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI ti_int __addvti3(ti_int a, ti_int b) { + ti_int s = (tu_int)a + (tu_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/apple_versioning.c b/lib/builtins/apple_versioning.c index 3797a1ab0..f87b42820 100644 --- a/lib/builtins/apple_versioning.c +++ b/lib/builtins/apple_versioning.c @@ -1,47 +1,42 @@ -/* ===-- apple_versioning.c - Adds versioning symbols for ld ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ - +//===-- apple_versioning.c - Adds versioning symbols for ld ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #if __APPLE__ - #include <Availability.h> - - #if __IPHONE_OS_VERSION_MIN_REQUIRED - #define NOT_HERE_BEFORE_10_6(sym) - #define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ - extern const char sym##_tmp61 __asm("$ld$hide$os6.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp61 = 0; \ - extern const char sym##_tmp60 __asm("$ld$hide$os6.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp60 = 0; \ - extern const char sym##_tmp51 __asm("$ld$hide$os5.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp51 = 0; \ - extern const char sym##_tmp50 __asm("$ld$hide$os5.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp50 = 0; - #else - #define NOT_HERE_BEFORE_10_6(sym) \ - extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ - extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp5 = 0; - #define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ - extern const char sym##_tmp8 __asm("$ld$hide$os10.8$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp8 = 0; \ - extern const char sym##_tmp7 __asm("$ld$hide$os10.7$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp7 = 0; \ - extern const char sym##_tmp6 __asm("$ld$hide$os10.6$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp6 = 0; - #endif +#include <Availability.h> +#if __IPHONE_OS_VERSION_MIN_REQUIRED +#define NOT_HERE_BEFORE_10_6(sym) +#define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ + extern const char sym##_tmp61 __asm("$ld$hide$os6.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp61 = 0; \ + extern const char sym##_tmp60 __asm("$ld$hide$os6.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp60 = 0; \ + extern const char sym##_tmp51 __asm("$ld$hide$os5.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp51 = 0; \ + extern const char sym##_tmp50 __asm("$ld$hide$os5.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp50 = 0; +#else +#define NOT_HERE_BEFORE_10_6(sym) \ + extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ + extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp5 = 0; +#define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ + extern const char sym##_tmp8 __asm("$ld$hide$os10.8$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp8 = 0; \ + extern const char sym##_tmp7 __asm("$ld$hide$os10.7$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp7 = 0; \ + extern const char sym##_tmp6 __asm("$ld$hide$os10.6$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp6 = 0; +#endif -/* Symbols in libSystem.dylib in 10.6 and later, - * but are in libgcc_s.dylib in earlier versions - */ +// Symbols in libSystem.dylib in 10.6 and later, +// but are in libgcc_s.dylib in earlier versions NOT_HERE_BEFORE_10_6(__absvdi2) NOT_HERE_BEFORE_10_6(__absvsi2) @@ -143,14 +138,13 @@ NOT_HERE_BEFORE_10_6(__udivti3) NOT_HERE_BEFORE_10_6(__umoddi3) NOT_HERE_BEFORE_10_6(__umodti3) - #if __ppc__ NOT_HERE_BEFORE_10_6(__gcc_qadd) NOT_HERE_BEFORE_10_6(__gcc_qdiv) NOT_HERE_BEFORE_10_6(__gcc_qmul) NOT_HERE_BEFORE_10_6(__gcc_qsub) NOT_HERE_BEFORE_10_6(__trampoline_setup) -#endif /* __ppc__ */ +#endif // __ppc__ NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange_1) @@ -201,24 +195,23 @@ NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_2) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_4) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_8) - #if __arm__ && __DYNAMIC__ - #define NOT_HERE_UNTIL_AFTER_4_3(sym) \ - extern const char sym##_tmp1 __asm("$ld$hide$os3.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp1 = 0; \ - extern const char sym##_tmp2 __asm("$ld$hide$os3.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp2 = 0; \ - extern const char sym##_tmp3 __asm("$ld$hide$os3.2$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp3 = 0; \ - extern const char sym##_tmp4 __asm("$ld$hide$os4.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ - extern const char sym##_tmp5 __asm("$ld$hide$os4.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp5 = 0; \ - extern const char sym##_tmp6 __asm("$ld$hide$os4.2$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp6 = 0; \ - extern const char sym##_tmp7 __asm("$ld$hide$os4.3$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp7 = 0; - +#define NOT_HERE_UNTIL_AFTER_4_3(sym) \ + extern const char sym##_tmp1 __asm("$ld$hide$os3.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp1 = 0; \ + extern const char sym##_tmp2 __asm("$ld$hide$os3.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp2 = 0; \ + extern const char sym##_tmp3 __asm("$ld$hide$os3.2$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp3 = 0; \ + extern const char sym##_tmp4 __asm("$ld$hide$os4.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ + extern const char sym##_tmp5 __asm("$ld$hide$os4.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp5 = 0; \ + extern const char sym##_tmp6 __asm("$ld$hide$os4.2$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp6 = 0; \ + extern const char sym##_tmp7 __asm("$ld$hide$os4.3$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp7 = 0; + NOT_HERE_UNTIL_AFTER_4_3(__absvdi2) NOT_HERE_UNTIL_AFTER_4_3(__absvsi2) NOT_HERE_UNTIL_AFTER_4_3(__adddf3) @@ -339,12 +332,8 @@ NOT_HERE_UNTIL_AFTER_4_3(__divmodsi4) NOT_HERE_UNTIL_AFTER_4_3(__udivmodsi4) #endif // __arm__ && __DYNAMIC__ - - - - -#else /* !__APPLE__ */ +#else // !__APPLE__ extern int avoid_empty_file; -#endif /* !__APPLE__*/ +#endif // !__APPLE__ diff --git a/lib/builtins/arm/adddf3vfp.S b/lib/builtins/arm/adddf3vfp.S index 8e476cad1..1a271db08 100644 --- a/lib/builtins/arm/adddf3vfp.S +++ b/lib/builtins/arm/adddf3vfp.S @@ -1,20 +1,18 @@ //===-- adddf3vfp.S - Implement adddf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // double __adddf3vfp(double a, double b) { return a + b; } // // Adds two double precision floating point numbers using the Darwin // calling convention where double arguments are passsed in GPR pairs -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) @@ -23,7 +21,7 @@ DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) #else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vadd.f64 d6, d6, d7 + vadd.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair #endif bx lr diff --git a/lib/builtins/arm/addsf3.S b/lib/builtins/arm/addsf3.S index 74723cbef..aa4d40473 100644 --- a/lib/builtins/arm/addsf3.S +++ b/lib/builtins/arm/addsf3.S @@ -1,17 +1,16 @@ -/*===-- addsf3.S - Adds two single precision floating pointer numbers-----===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __addsf3 (single precision floating pointer number - * addition with the IEEE-754 default rounding (to nearest, ties to even) - * function for the ARM Thumb1 ISA. - * - *===----------------------------------------------------------------------===*/ +//===-- addsf3.S - Adds two single precision floating pointer numbers-----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __addsf3 (single precision floating pointer number +// addition with the IEEE-754 default rounding (to nearest, ties to even) +// function for the ARM Thumb1 ISA. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" #define significandBits 23 @@ -29,9 +28,9 @@ DEFINE_COMPILERRT_THUMB_FUNCTION(__addsf3) // Get the absolute value of a and b. lsls r2, r0, #1 lsls r3, r1, #1 - lsrs r2, r2, #1 /* aAbs */ + lsrs r2, r2, #1 // aAbs beq LOCAL_LABEL(a_zero_nan_inf) - lsrs r3, r3, #1 /* bAbs */ + lsrs r3, r3, #1 // bAbs beq LOCAL_LABEL(zero_nan_inf) // Detect if a or b is infinity or Nan. @@ -55,9 +54,9 @@ LOCAL_LABEL(no_swap): // Get the significands and shift them to give us round, guard and sticky. lsls r4, r0, #(typeWidth - significandBits) - lsrs r4, r4, #(typeWidth - significandBits - 3) /* aSignificand << 3 */ + lsrs r4, r4, #(typeWidth - significandBits - 3) // aSignificand << 3 lsls r5, r1, #(typeWidth - significandBits) - lsrs r5, r5, #(typeWidth - significandBits - 3) /* bSignificand << 3 */ + lsrs r5, r5, #(typeWidth - significandBits - 3) // bSignificand << 3 // Get the implicitBit. movs r6, #1 @@ -199,7 +198,7 @@ LOCAL_LABEL(do_substraction): beq 1f movs r7, #1 1: - lsrs r4, r6 /* aSignificand >> shift */ + lsrs r4, r6 // aSignificand >> shift orrs r4, r7 b LOCAL_LABEL(form_result) diff --git a/lib/builtins/arm/addsf3vfp.S b/lib/builtins/arm/addsf3vfp.S index 8871efdcc..c9d1fd150 100644 --- a/lib/builtins/arm/addsf3vfp.S +++ b/lib/builtins/arm/addsf3vfp.S @@ -1,9 +1,8 @@ //===-- addsf3vfp.S - Implement addsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cdcmp.S b/lib/builtins/arm/aeabi_cdcmp.S index adc2d55d9..bd039a032 100644 --- a/lib/builtins/arm/aeabi_cdcmp.S +++ b/lib/builtins/arm/aeabi_cdcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_cdcmp.S - EABI cdcmp* implementation ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c index 7578433a1..7bae8743f 100644 --- a/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c +++ b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c @@ -1,16 +1,15 @@ //===-- lib/arm/aeabi_cdcmpeq_helper.c - Helper for cdcmpeq ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include <stdint.h> #include "../int_lib.h" +#include <stdint.h> -AEABI_RTABI __attribute__((visibility("hidden"))) -int __aeabi_cdcmpeq_check_nan(double a, double b) { - return __builtin_isnan(a) || __builtin_isnan(b); +AEABI_RTABI __attribute__((visibility("hidden"))) int +__aeabi_cdcmpeq_check_nan(double a, double b) { + return __builtin_isnan(a) || __builtin_isnan(b); } diff --git a/lib/builtins/arm/aeabi_cfcmp.S b/lib/builtins/arm/aeabi_cfcmp.S index 4b1de9976..a26cb2a3c 100644 --- a/lib/builtins/arm/aeabi_cfcmp.S +++ b/lib/builtins/arm/aeabi_cfcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_cfcmp.S - EABI cfcmp* implementation ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c index 43dde9a49..25407337d 100644 --- a/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c +++ b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c @@ -1,16 +1,15 @@ //===-- lib/arm/aeabi_cfcmpeq_helper.c - Helper for cdcmpeq ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include <stdint.h> #include "../int_lib.h" +#include <stdint.h> -AEABI_RTABI __attribute__((visibility("hidden"))) -int __aeabi_cfcmpeq_check_nan(float a, float b) { - return __builtin_isnan(a) || __builtin_isnan(b); +AEABI_RTABI __attribute__((visibility("hidden"))) int +__aeabi_cfcmpeq_check_nan(float a, float b) { + return __builtin_isnan(a) || __builtin_isnan(b); } diff --git a/lib/builtins/arm/aeabi_dcmp.S b/lib/builtins/arm/aeabi_dcmp.S index 9fa78b461..5f720670d 100644 --- a/lib/builtins/arm/aeabi_dcmp.S +++ b/lib/builtins/arm/aeabi_dcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_dcmp.S - EABI dcmp* implementation ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_div0.c b/lib/builtins/arm/aeabi_div0.c index dc3031326..7e8862321 100644 --- a/lib/builtins/arm/aeabi_div0.c +++ b/lib/builtins/arm/aeabi_div0.c @@ -1,34 +1,30 @@ -/* ===-- aeabi_div0.c - ARM Runtime ABI support routines for compiler-rt ---=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements the division by zero helper routines as specified by the - * Run-time ABI for the ARM Architecture. - * - * ===----------------------------------------------------------------------=== - */ +//===-- aeabi_div0.c - ARM Runtime ABI support routines for compiler-rt ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the division by zero helper routines as specified by the +// Run-time ABI for the ARM Architecture. +// +//===----------------------------------------------------------------------===// -/* - * RTABI 4.3.2 - Division by zero - * - * The *div0 functions: - * - Return the value passed to them as a parameter - * - Or, return a fixed value defined by the execution environment (such as 0) - * - Or, raise a signal (often SIGFPE) or throw an exception, and do not return - * - * An application may provide its own implementations of the *div0 functions to - * for a particular behaviour from the *div and *divmod functions called out of - * line. - */ +// RTABI 4.3.2 - Division by zero +// +// The *div0 functions: +// - Return the value passed to them as a parameter +// - Or, return a fixed value defined by the execution environment (such as 0) +// - Or, raise a signal (often SIGFPE) or throw an exception, and do not return +// +// An application may provide its own implementations of the *div0 functions to +// for a particular behaviour from the *div and *divmod functions called out of +// line. #include "../int_lib.h" -/* provide an unused declaration to pacify pendantic compilation */ +// provide an unused declaration to pacify pendantic compilation extern unsigned char declaration; #if defined(__ARM_EABI__) @@ -37,9 +33,8 @@ __aeabi_idiv0(int return_value) { return return_value; } -AEABI_RTABI long long __attribute__((weak)) __attribute__((visibility("hidden"))) -__aeabi_ldiv0(long long return_value) { +AEABI_RTABI long long __attribute__((weak)) +__attribute__((visibility("hidden"))) __aeabi_ldiv0(long long return_value) { return return_value; } #endif - diff --git a/lib/builtins/arm/aeabi_drsub.c b/lib/builtins/arm/aeabi_drsub.c index 125488608..e4e8dc051 100644 --- a/lib/builtins/arm/aeabi_drsub.c +++ b/lib/builtins/arm/aeabi_drsub.c @@ -1,19 +1,14 @@ //===-- lib/arm/aeabi_drsub.c - Double-precision subtraction --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "../fp_lib.h" -AEABI_RTABI fp_t -__aeabi_dsub(fp_t, fp_t); +AEABI_RTABI fp_t __aeabi_dsub(fp_t, fp_t); -AEABI_RTABI fp_t -__aeabi_drsub(fp_t a, fp_t b) { - return __aeabi_dsub(b, a); -} +AEABI_RTABI fp_t __aeabi_drsub(fp_t a, fp_t b) { return __aeabi_dsub(b, a); } diff --git a/lib/builtins/arm/aeabi_fcmp.S b/lib/builtins/arm/aeabi_fcmp.S index ea5b96c21..cd311b417 100644 --- a/lib/builtins/arm/aeabi_fcmp.S +++ b/lib/builtins/arm/aeabi_fcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_fcmp.S - EABI fcmp* implementation ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_frsub.c b/lib/builtins/arm/aeabi_frsub.c index 34f230374..9a363248f 100644 --- a/lib/builtins/arm/aeabi_frsub.c +++ b/lib/builtins/arm/aeabi_frsub.c @@ -1,19 +1,14 @@ //===-- lib/arm/aeabi_frsub.c - Single-precision subtraction --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "../fp_lib.h" -AEABI_RTABI fp_t -__aeabi_fsub(fp_t, fp_t); +AEABI_RTABI fp_t __aeabi_fsub(fp_t, fp_t); -AEABI_RTABI fp_t -__aeabi_frsub(fp_t a, fp_t b) { - return __aeabi_fsub(b, a); -} +AEABI_RTABI fp_t __aeabi_frsub(fp_t a, fp_t b) { return __aeabi_fsub(b, a); } diff --git a/lib/builtins/arm/aeabi_idivmod.S b/lib/builtins/arm/aeabi_idivmod.S index 9c9c80ab5..bb80e4b96 100644 --- a/lib/builtins/arm/aeabi_idivmod.S +++ b/lib/builtins/arm/aeabi_idivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_idivmod.S - EABI idivmod implementation ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_ldivmod.S b/lib/builtins/arm/aeabi_ldivmod.S index 038ae5d72..d0d06be6f 100644 --- a/lib/builtins/arm/aeabi_ldivmod.S +++ b/lib/builtins/arm/aeabi_ldivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_ldivmod.S - EABI ldivmod implementation ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memcmp.S b/lib/builtins/arm/aeabi_memcmp.S index e86d61137..41637289c 100644 --- a/lib/builtins/arm/aeabi_memcmp.S +++ b/lib/builtins/arm/aeabi_memcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_memcmp.S - EABI memcmp implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memcpy.S b/lib/builtins/arm/aeabi_memcpy.S index e83c5fd4d..93e1b05d5 100644 --- a/lib/builtins/arm/aeabi_memcpy.S +++ b/lib/builtins/arm/aeabi_memcpy.S @@ -1,9 +1,8 @@ //===-- aeabi_memcpy.S - EABI memcpy implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memmove.S b/lib/builtins/arm/aeabi_memmove.S index ee28300e4..c2f0fa4cd 100644 --- a/lib/builtins/arm/aeabi_memmove.S +++ b/lib/builtins/arm/aeabi_memmove.S @@ -1,9 +1,8 @@ //===-- aeabi_memmove.S - EABI memmove implementation --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memset.S b/lib/builtins/arm/aeabi_memset.S index 0a678d762..2aa8ec0c4 100644 --- a/lib/builtins/arm/aeabi_memset.S +++ b/lib/builtins/arm/aeabi_memset.S @@ -1,9 +1,8 @@ //===-- aeabi_memset.S - EABI memset implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_uidivmod.S b/lib/builtins/arm/aeabi_uidivmod.S index 88a4a6d8b..df030769f 100644 --- a/lib/builtins/arm/aeabi_uidivmod.S +++ b/lib/builtins/arm/aeabi_uidivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_uidivmod.S - EABI uidivmod implementation -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_uldivmod.S b/lib/builtins/arm/aeabi_uldivmod.S index be343b6bc..4fc97704d 100644 --- a/lib/builtins/arm/aeabi_uldivmod.S +++ b/lib/builtins/arm/aeabi_uldivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_uldivmod.S - EABI uldivmod implementation -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/bswapdi2.S b/lib/builtins/arm/bswapdi2.S index e9db8bac7..271df8bd6 100644 --- a/lib/builtins/arm/bswapdi2.S +++ b/lib/builtins/arm/bswapdi2.S @@ -1,9 +1,8 @@ //===------- bswapdi2 - Implement bswapdi2 --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/bswapsi2.S b/lib/builtins/arm/bswapsi2.S index 1f6eed5c1..07cc3d891 100644 --- a/lib/builtins/arm/bswapsi2.S +++ b/lib/builtins/arm/bswapsi2.S @@ -1,9 +1,8 @@ //===------- bswapsi2 - Implement bswapsi2 --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/chkstk.S b/lib/builtins/arm/chkstk.S index e30021058..c5c9ebe0a 100644 --- a/lib/builtins/arm/chkstk.S +++ b/lib/builtins/arm/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/arm/clzdi2.S b/lib/builtins/arm/clzdi2.S index fc03b385c..685668b11 100644 --- a/lib/builtins/arm/clzdi2.S +++ b/lib/builtins/arm/clzdi2.S @@ -1,16 +1,15 @@ -/* ===-- clzdi2.c - Implement __clzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements count leading zeros for 64bit arguments. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzdi2.c - Implement __clzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements count leading zeros for 64bit arguments. +// +//===----------------------------------------------------------------------===// + #include "../assembly.h" .syntax unified @@ -35,14 +34,12 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) #endif JMP(lr) #else - /* Assumption: n != 0 */ + // Assumption: n != 0 - /* - * r0: n - * r1: upper half of n, overwritten after check - * r1: count of leading zeros in n + 1 - * r2: scratch register for shifted r0 - */ + // r0: n + // r1: upper half of n, overwritten after check + // r1: count of leading zeros in n + 1 + // r2: scratch register for shifted r0 #ifdef __ARMEB__ cmp r0, 0 moveq r0, r1 @@ -53,14 +50,12 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) movne r1, 1 moveq r1, 33 - /* - * Basic block: - * if ((r0 >> SHIFT) == 0) - * r1 += SHIFT; - * else - * r0 >>= SHIFT; - * for descending powers of two as SHIFT. - */ + // Basic block: + // if ((r0 >> SHIFT) == 0) + // r1 += SHIFT; + // else + // r0 >>= SHIFT; + // for descending powers of two as SHIFT. #define BLOCK(shift) \ lsrs r2, r0, shift; \ movne r0, r2; \ @@ -71,18 +66,16 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) BLOCK(4) BLOCK(2) - /* - * The basic block invariants at this point are (r0 >> 2) == 0 and - * r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. - * - * r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) - * ---+----------------+----------------+------------+-------------- - * 1 | 1 | 0 | 0 | 1 - * 2 | 0 | 1 | -1 | 0 - * 3 | 0 | 1 | -1 | 0 - * - * The r1's initial value of 1 compensates for the 1 here. - */ + // The basic block invariants at this point are (r0 >> 2) == 0 and + // r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. + // + // r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) + // ---+----------------+----------------+------------+-------------- + // 1 | 1 | 0 | 0 | 1 + // 2 | 0 | 1 | -1 | 0 + // 3 | 0 | 1 | -1 | 0 + // + // The r1's initial value of 1 compensates for the 1 here. sub r0, r1, r0, lsr #1 JMP(lr) diff --git a/lib/builtins/arm/clzsi2.S b/lib/builtins/arm/clzsi2.S index f2ce59c90..5d86fe486 100644 --- a/lib/builtins/arm/clzsi2.S +++ b/lib/builtins/arm/clzsi2.S @@ -1,16 +1,15 @@ -/* ===-- clzsi2.c - Implement __clzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements count leading zeros for 32bit arguments. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzsi2.c - Implement __clzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements count leading zeros for 32bit arguments. +// +//===----------------------------------------------------------------------===// + #include "../assembly.h" .syntax unified @@ -23,23 +22,19 @@ DEFINE_COMPILERRT_FUNCTION(__clzsi2) clz r0, r0 JMP(lr) #else - /* Assumption: n != 0 */ + // Assumption: n != 0 - /* - * r0: n - * r1: count of leading zeros in n + 1 - * r2: scratch register for shifted r0 - */ + // r0: n + // r1: count of leading zeros in n + 1 + // r2: scratch register for shifted r0 mov r1, 1 - /* - * Basic block: - * if ((r0 >> SHIFT) == 0) - * r1 += SHIFT; - * else - * r0 >>= SHIFT; - * for descending powers of two as SHIFT. - */ + // Basic block: + // if ((r0 >> SHIFT) == 0) + // r1 += SHIFT; + // else + // r0 >>= SHIFT; + // for descending powers of two as SHIFT. #define BLOCK(shift) \ lsrs r2, r0, shift; \ @@ -51,18 +46,16 @@ DEFINE_COMPILERRT_FUNCTION(__clzsi2) BLOCK(4) BLOCK(2) - /* - * The basic block invariants at this point are (r0 >> 2) == 0 and - * r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. - * - * r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) - * ---+----------------+----------------+------------+-------------- - * 1 | 1 | 0 | 0 | 1 - * 2 | 0 | 1 | -1 | 0 - * 3 | 0 | 1 | -1 | 0 - * - * The r1's initial value of 1 compensates for the 1 here. - */ + // The basic block invariants at this point are (r0 >> 2) == 0 and + // r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. + // + // r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) + // ---+----------------+----------------+------------+-------------- + // 1 | 1 | 0 | 0 | 1 + // 2 | 0 | 1 | -1 | 0 + // 3 | 0 | 1 | -1 | 0 + // + // The r1's initial value of 1 compensates for the 1 here. sub r0, r1, r0, lsr #1 JMP(lr) diff --git a/lib/builtins/arm/comparesf2.S b/lib/builtins/arm/comparesf2.S index c6c4cc067..d5cc9228b 100644 --- a/lib/builtins/arm/comparesf2.S +++ b/lib/builtins/arm/comparesf2.S @@ -1,9 +1,8 @@ //===-- comparesf2.S - Implement single-precision soft-float comparisons --===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -176,6 +175,11 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__lesf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__ltsf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) +#if defined(__ELF__) +// Alias for libgcc compatibility +DEFINE_COMPILERRT_FUNCTION_ALIAS(__cmpsf2, __lesf2) +#endif + @ int __gtsf2(float a, float b) .p2align 2 diff --git a/lib/builtins/arm/divdf3vfp.S b/lib/builtins/arm/divdf3vfp.S index 776ba4f24..ad50b57a6 100644 --- a/lib/builtins/arm/divdf3vfp.S +++ b/lib/builtins/arm/divdf3vfp.S @@ -1,9 +1,8 @@ //===-- divdf3vfp.S - Implement divdf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/divmodsi4.S b/lib/builtins/arm/divmodsi4.S index 8a027b741..f94438dfd 100644 --- a/lib/builtins/arm/divmodsi4.S +++ b/lib/builtins/arm/divmodsi4.S @@ -1,17 +1,16 @@ -/*===-- divmodsi4.S - 32-bit signed integer divide and modulus ------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __divmodsi4 (32-bit signed integer divide and - * modulus) function for the ARM architecture. A naive digit-by-digit - * computation is employed for simplicity. - * - *===----------------------------------------------------------------------===*/ +//===-- divmodsi4.S - 32-bit signed integer divide and modulus ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __divmodsi4 (32-bit signed integer divide and +// modulus) function for the ARM architecture. A naive digit-by-digit +// computation is employed for simplicity. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/divsf3vfp.S b/lib/builtins/arm/divsf3vfp.S index 130318f0c..958a6724b 100644 --- a/lib/builtins/arm/divsf3vfp.S +++ b/lib/builtins/arm/divsf3vfp.S @@ -1,9 +1,8 @@ //===-- divsf3vfp.S - Implement divsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/divsi3.S b/lib/builtins/arm/divsi3.S index 19757af17..761bf49d3 100644 --- a/lib/builtins/arm/divsi3.S +++ b/lib/builtins/arm/divsi3.S @@ -1,16 +1,15 @@ -/*===-- divsi3.S - 32-bit signed integer divide ---------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __divsi3 (32-bit signed integer divide) function - * for the ARM architecture as a wrapper around the unsigned routine. - * - *===----------------------------------------------------------------------===*/ +//===-- divsi3.S - 32-bit signed integer divide ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __divsi3 (32-bit signed integer divide) function +// for the ARM architecture as a wrapper around the unsigned routine. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/eqdf2vfp.S b/lib/builtins/arm/eqdf2vfp.S index d50706570..2a0a64b97 100644 --- a/lib/builtins/arm/eqdf2vfp.S +++ b/lib/builtins/arm/eqdf2vfp.S @@ -1,21 +1,19 @@ //===-- eqdf2vfp.S - Implement eqdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // extern int __eqdf2vfp(double a, double b); // // Returns one iff a == b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) @@ -24,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) #else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 #endif vmrs apsr_nzcv, fpscr ITE(eq) diff --git a/lib/builtins/arm/eqsf2vfp.S b/lib/builtins/arm/eqsf2vfp.S index fd72b2fdb..5fefe7b71 100644 --- a/lib/builtins/arm/eqsf2vfp.S +++ b/lib/builtins/arm/eqsf2vfp.S @@ -1,9 +1,8 @@ //===-- eqsf2vfp.S - Implement eqsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __eqsf2vfp(float a, float b); // // Returns one iff a == b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/extendsfdf2vfp.S b/lib/builtins/arm/extendsfdf2vfp.S index 1079f977b..37c8be8dc 100644 --- a/lib/builtins/arm/extendsfdf2vfp.S +++ b/lib/builtins/arm/extendsfdf2vfp.S @@ -1,9 +1,8 @@ //===-- extendsfdf2vfp.S - Implement extendsfdf2vfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __extendsfdf2vfp(float a); // // Converts single precision float to double precision result. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR and a double precision result is returned in R0/R1 pair. // .syntax unified diff --git a/lib/builtins/arm/fixdfsivfp.S b/lib/builtins/arm/fixdfsivfp.S index 5d7b0f856..af1d4f4fa 100644 --- a/lib/builtins/arm/fixdfsivfp.S +++ b/lib/builtins/arm/fixdfsivfp.S @@ -1,9 +1,8 @@ //===-- fixdfsivfp.S - Implement fixdfsivfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __fixdfsivfp(double a); // // Converts double precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/fixsfsivfp.S b/lib/builtins/arm/fixsfsivfp.S index 805a277af..30b2f3cf9 100644 --- a/lib/builtins/arm/fixsfsivfp.S +++ b/lib/builtins/arm/fixsfsivfp.S @@ -1,9 +1,8 @@ //===-- fixsfsivfp.S - Implement fixsfsivfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __fixsfsivfp(float a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/fixunsdfsivfp.S b/lib/builtins/arm/fixunsdfsivfp.S index 4f1b2c8ce..44e6dbd49 100644 --- a/lib/builtins/arm/fixunsdfsivfp.S +++ b/lib/builtins/arm/fixunsdfsivfp.S @@ -1,9 +1,8 @@ //===-- fixunsdfsivfp.S - Implement fixunsdfsivfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,9 +11,9 @@ // // extern unsigned int __fixunsdfsivfp(double a); // -// Converts double precision float to a 32-bit unsigned int rounding towards +// Converts double precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/fixunssfsivfp.S b/lib/builtins/arm/fixunssfsivfp.S index e5d778236..5d6ee7cce 100644 --- a/lib/builtins/arm/fixunssfsivfp.S +++ b/lib/builtins/arm/fixunssfsivfp.S @@ -1,9 +1,8 @@ //===-- fixunssfsivfp.S - Implement fixunssfsivfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,9 +11,9 @@ // // extern unsigned int __fixunssfsivfp(float a); // -// Converts single precision float to a 32-bit unsigned int rounding towards +// Converts single precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/floatsidfvfp.S b/lib/builtins/arm/floatsidfvfp.S index 3297ad44d..ae8d24658 100644 --- a/lib/builtins/arm/floatsidfvfp.S +++ b/lib/builtins/arm/floatsidfvfp.S @@ -1,9 +1,8 @@ //===-- floatsidfvfp.S - Implement floatsidfvfp ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __floatsidfvfp(int a); // // Converts a 32-bit int to a double precision float. -// Uses Darwin calling convention where a double precision result is +// Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/floatsisfvfp.S b/lib/builtins/arm/floatsisfvfp.S index 65408b54b..a36bc5ee5 100644 --- a/lib/builtins/arm/floatsisfvfp.S +++ b/lib/builtins/arm/floatsisfvfp.S @@ -1,9 +1,8 @@ //===-- floatsisfvfp.S - Implement floatsisfvfp ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __floatsisfvfp(int a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision result is +// Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/floatunssidfvfp.S b/lib/builtins/arm/floatunssidfvfp.S index d7a7024a2..0932dab2b 100644 --- a/lib/builtins/arm/floatunssidfvfp.S +++ b/lib/builtins/arm/floatunssidfvfp.S @@ -1,9 +1,8 @@ //===-- floatunssidfvfp.S - Implement floatunssidfvfp ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __floatunssidfvfp(unsigned int a); // // Converts a 32-bit int to a double precision float. -// Uses Darwin calling convention where a double precision result is +// Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/floatunssisfvfp.S b/lib/builtins/arm/floatunssisfvfp.S index 1ca856519..9578546f4 100644 --- a/lib/builtins/arm/floatunssisfvfp.S +++ b/lib/builtins/arm/floatunssisfvfp.S @@ -1,9 +1,8 @@ //===-- floatunssisfvfp.S - Implement floatunssisfvfp ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __floatunssisfvfp(unsigned int a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision result is +// Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/gedf2vfp.S b/lib/builtins/arm/gedf2vfp.S index 364fc5b24..2af9d9099 100644 --- a/lib/builtins/arm/gedf2vfp.S +++ b/lib/builtins/arm/gedf2vfp.S @@ -1,9 +1,8 @@ //===-- gedf2vfp.S - Implement gedf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gedf2vfp(double a, double b); // // Returns one iff a >= b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/gesf2vfp.S b/lib/builtins/arm/gesf2vfp.S index 346c3473a..cedd1e13e 100644 --- a/lib/builtins/arm/gesf2vfp.S +++ b/lib/builtins/arm/gesf2vfp.S @@ -1,9 +1,8 @@ //===-- gesf2vfp.S - Implement gesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gesf2vfp(float a, float b); // // Returns one iff a >= b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/gtdf2vfp.S b/lib/builtins/arm/gtdf2vfp.S index 3389c3ad9..782ad8cac 100644 --- a/lib/builtins/arm/gtdf2vfp.S +++ b/lib/builtins/arm/gtdf2vfp.S @@ -1,9 +1,8 @@ //===-- gtdf2vfp.S - Implement gtdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __gtdf2vfp(double a, double b); // // Returns one iff a > b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/gtsf2vfp.S b/lib/builtins/arm/gtsf2vfp.S index afdba8b01..1cc2bd14f 100644 --- a/lib/builtins/arm/gtsf2vfp.S +++ b/lib/builtins/arm/gtsf2vfp.S @@ -1,9 +1,8 @@ //===-- gtsf2vfp.S - Implement gtsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gtsf2vfp(float a, float b); // // Returns one iff a > b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/ledf2vfp.S b/lib/builtins/arm/ledf2vfp.S index 4bbe4c868..0097e4b6c 100644 --- a/lib/builtins/arm/ledf2vfp.S +++ b/lib/builtins/arm/ledf2vfp.S @@ -1,9 +1,8 @@ //===-- ledf2vfp.S - Implement ledf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __ledf2vfp(double a, double b); // // Returns one iff a <= b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/lesf2vfp.S b/lib/builtins/arm/lesf2vfp.S index 51232bd8c..2052d3869 100644 --- a/lib/builtins/arm/lesf2vfp.S +++ b/lib/builtins/arm/lesf2vfp.S @@ -1,9 +1,8 @@ //===-- lesf2vfp.S - Implement lesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __lesf2vfp(float a, float b); // // Returns one iff a <= b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/ltdf2vfp.S b/lib/builtins/arm/ltdf2vfp.S index 8e2928c81..a126aa9e0 100644 --- a/lib/builtins/arm/ltdf2vfp.S +++ b/lib/builtins/arm/ltdf2vfp.S @@ -1,9 +1,8 @@ //===-- ltdf2vfp.S - Implement ltdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __ltdf2vfp(double a, double b); // // Returns one iff a < b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/ltsf2vfp.S b/lib/builtins/arm/ltsf2vfp.S index 59c00c6ba..ba10d71d2 100644 --- a/lib/builtins/arm/ltsf2vfp.S +++ b/lib/builtins/arm/ltsf2vfp.S @@ -1,9 +1,8 @@ //===-- ltsf2vfp.S - Implement ltsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __ltsf2vfp(float a, float b); // // Returns one iff a < b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/modsi3.S b/lib/builtins/arm/modsi3.S index be263834d..5312f5b41 100644 --- a/lib/builtins/arm/modsi3.S +++ b/lib/builtins/arm/modsi3.S @@ -1,16 +1,15 @@ -/*===-- modsi3.S - 32-bit signed integer modulus --------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __modsi3 (32-bit signed integer modulus) function - * for the ARM architecture as a wrapper around the unsigned routine. - * - *===----------------------------------------------------------------------===*/ +//===-- modsi3.S - 32-bit signed integer modulus --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __modsi3 (32-bit signed integer modulus) function +// for the ARM architecture as a wrapper around the unsigned routine. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/muldf3vfp.S b/lib/builtins/arm/muldf3vfp.S index aa7b23495..9adc937bc 100644 --- a/lib/builtins/arm/muldf3vfp.S +++ b/lib/builtins/arm/muldf3vfp.S @@ -1,9 +1,8 @@ //===-- muldf3vfp.S - Implement muldf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/mulsf3vfp.S b/lib/builtins/arm/mulsf3vfp.S index a1da789dc..a94131beb 100644 --- a/lib/builtins/arm/mulsf3vfp.S +++ b/lib/builtins/arm/mulsf3vfp.S @@ -1,9 +1,8 @@ //===-- mulsf3vfp.S - Implement mulsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/nedf2vfp.S b/lib/builtins/arm/nedf2vfp.S index aef72eb00..32d35c41d 100644 --- a/lib/builtins/arm/nedf2vfp.S +++ b/lib/builtins/arm/nedf2vfp.S @@ -1,21 +1,19 @@ //===-- nedf2vfp.S - Implement nedf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // extern double __nedf2vfp(double a, double b); // // Returns zero if a and b are unequal and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) @@ -24,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) #else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 #endif vmrs apsr_nzcv, fpscr ITE(ne) diff --git a/lib/builtins/arm/negdf2vfp.S b/lib/builtins/arm/negdf2vfp.S index 81f0ab8ee..b7cf91877 100644 --- a/lib/builtins/arm/negdf2vfp.S +++ b/lib/builtins/arm/negdf2vfp.S @@ -1,9 +1,8 @@ //===-- negdf2vfp.S - Implement negdf2vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern double __negdf2vfp(double a, double b); // -// Returns the negation a double precision floating point numbers using the +// Returns the negation a double precision floating point numbers using the // Darwin calling convention where double arguments are passsed in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/negsf2vfp.S b/lib/builtins/arm/negsf2vfp.S index 46ab4a9cf..b6d3c6189 100644 --- a/lib/builtins/arm/negsf2vfp.S +++ b/lib/builtins/arm/negsf2vfp.S @@ -1,9 +1,8 @@ //===-- negsf2vfp.S - Implement negsf2vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern float __negsf2vfp(float a); // -// Returns the negation of a single precision floating point numbers using the +// Returns the negation of a single precision floating point numbers using the // Darwin calling convention where single arguments are passsed like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/nesf2vfp.S b/lib/builtins/arm/nesf2vfp.S index 50d60f493..34c8bb489 100644 --- a/lib/builtins/arm/nesf2vfp.S +++ b/lib/builtins/arm/nesf2vfp.S @@ -1,9 +1,8 @@ //===-- nesf2vfp.S - Implement nesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __nesf2vfp(float a, float b); // // Returns one iff a != b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/restore_vfp_d8_d15_regs.S b/lib/builtins/arm/restore_vfp_d8_d15_regs.S index 0692cf3e1..fd6d59bb3 100644 --- a/lib/builtins/arm/restore_vfp_d8_d15_regs.S +++ b/lib/builtins/arm/restore_vfp_d8_d15_regs.S @@ -1,9 +1,8 @@ //===-- save_restore_regs.S - Implement save/restore* ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/save_vfp_d8_d15_regs.S b/lib/builtins/arm/save_vfp_d8_d15_regs.S index 544dd5467..5eb3a2fba 100644 --- a/lib/builtins/arm/save_vfp_d8_d15_regs.S +++ b/lib/builtins/arm/save_vfp_d8_d15_regs.S @@ -1,9 +1,8 @@ //===-- save_restore_regs.S - Implement save/restore* ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/softfloat-alias.list b/lib/builtins/arm/softfloat-alias.list index cc6a4b3cd..ab6ed21e5 100644 --- a/lib/builtins/arm/softfloat-alias.list +++ b/lib/builtins/arm/softfloat-alias.list @@ -1,5 +1,5 @@ # -# These are soft float functions which can be +# These are soft float functions which can be # aliased to the *vfp functions on arm processors # that support floating point instructions. # diff --git a/lib/builtins/arm/subdf3vfp.S b/lib/builtins/arm/subdf3vfp.S index 2b6f2bdbf..f4eaf9af1 100644 --- a/lib/builtins/arm/subdf3vfp.S +++ b/lib/builtins/arm/subdf3vfp.S @@ -1,9 +1,8 @@ //===-- subdf3vfp.S - Implement subdf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern double __subdf3vfp(double a, double b); // -// Returns difference between two double precision floating point numbers using +// Returns difference between two double precision floating point numbers using // the Darwin calling convention where double arguments are passsed in GPR pairs // .syntax unified @@ -23,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__subdf3vfp) #else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vsub.f64 d6, d6, d7 + vsub.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair #endif bx lr diff --git a/lib/builtins/arm/subsf3vfp.S b/lib/builtins/arm/subsf3vfp.S index 3e83ea265..80e69f2e8 100644 --- a/lib/builtins/arm/subsf3vfp.S +++ b/lib/builtins/arm/subsf3vfp.S @@ -1,9 +1,8 @@ //===-- subsf3vfp.S - Implement subsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch16.S b/lib/builtins/arm/switch16.S index df9e38e17..a4b568da5 100644 --- a/lib/builtins/arm/switch16.S +++ b/lib/builtins/arm/switch16.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch32.S b/lib/builtins/arm/switch32.S index d97b53614..f2a5af527 100644 --- a/lib/builtins/arm/switch32.S +++ b/lib/builtins/arm/switch32.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch8.S b/lib/builtins/arm/switch8.S index 4d9e0eaff..0db875c32 100644 --- a/lib/builtins/arm/switch8.S +++ b/lib/builtins/arm/switch8.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switchu8.S b/lib/builtins/arm/switchu8.S index 4ffe35f05..551abebc6 100644 --- a/lib/builtins/arm/switchu8.S +++ b/lib/builtins/arm/switchu8.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/sync-ops.h b/lib/builtins/arm/sync-ops.h index ee02c30c6..5bb863d8c 100644 --- a/lib/builtins/arm/sync-ops.h +++ b/lib/builtins/arm/sync-ops.h @@ -1,64 +1,61 @@ -/*===-- sync-ops.h - --===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements outline macros for the __sync_fetch_and_* - * operations. Different instantiations will generate appropriate assembly for - * ARM and Thumb-2 versions of the functions. - * - *===----------------------------------------------------------------------===*/ +//===-- sync-ops.h - --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements outline macros for the __sync_fetch_and_* +// operations. Different instantiations will generate appropriate assembly for +// ARM and Thumb-2 versions of the functions. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" -#define SYNC_OP_4(op) \ - .p2align 2 ; \ - .thumb ; \ - .syntax unified ; \ - DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_ ## op) \ - dmb ; \ - mov r12, r0 ; \ - LOCAL_LABEL(tryatomic_ ## op): \ - ldrex r0, [r12] ; \ - op(r2, r0, r1) ; \ - strex r3, r2, [r12] ; \ - cmp r3, #0 ; \ - bne LOCAL_LABEL(tryatomic_ ## op) ; \ - dmb ; \ - bx lr +#define SYNC_OP_4(op) \ + .p2align 2; \ + .thumb; \ + .syntax unified; \ + DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \ + dmb; \ + mov r12, r0; \ + LOCAL_LABEL(tryatomic_##op) : ldrex r0, [r12]; \ + op(r2, r0, r1); \ + strex r3, r2, [r12]; \ + cmp r3, #0; \ + bne LOCAL_LABEL(tryatomic_##op); \ + dmb; \ + bx lr -#define SYNC_OP_8(op) \ - .p2align 2 ; \ - .thumb ; \ - .syntax unified ; \ - DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_ ## op) \ - push {r4, r5, r6, lr} ; \ - dmb ; \ - mov r12, r0 ; \ - LOCAL_LABEL(tryatomic_ ## op): \ - ldrexd r0, r1, [r12] ; \ - op(r4, r5, r0, r1, r2, r3) ; \ - strexd r6, r4, r5, [r12] ; \ - cmp r6, #0 ; \ - bne LOCAL_LABEL(tryatomic_ ## op) ; \ - dmb ; \ - pop {r4, r5, r6, pc} +#define SYNC_OP_8(op) \ + .p2align 2; \ + .thumb; \ + .syntax unified; \ + DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \ + push{r4, r5, r6, lr}; \ + dmb; \ + mov r12, r0; \ + LOCAL_LABEL(tryatomic_##op) : ldrexd r0, r1, [r12]; \ + op(r4, r5, r0, r1, r2, r3); \ + strexd r6, r4, r5, [r12]; \ + cmp r6, #0; \ + bne LOCAL_LABEL(tryatomic_##op); \ + dmb; \ + pop { r4, r5, r6, pc } -#define MINMAX_4(rD, rN, rM, cmp_kind) \ - cmp rN, rM ; \ - mov rD, rM ; \ - it cmp_kind ; \ - mov##cmp_kind rD, rN +#define MINMAX_4(rD, rN, rM, cmp_kind) \ + cmp rN, rM; \ + mov rD, rM; \ + it cmp_kind; \ + mov##cmp_kind rD, rN -#define MINMAX_8(rD_LO, rD_HI, rN_LO, rN_HI, rM_LO, rM_HI, cmp_kind) \ - cmp rN_LO, rM_LO ; \ - sbcs rN_HI, rM_HI ; \ - mov rD_LO, rM_LO ; \ - mov rD_HI, rM_HI ; \ - itt cmp_kind ; \ - mov##cmp_kind rD_LO, rN_LO ; \ - mov##cmp_kind rD_HI, rN_HI +#define MINMAX_8(rD_LO, rD_HI, rN_LO, rN_HI, rM_LO, rM_HI, cmp_kind) \ + cmp rN_LO, rM_LO; \ + sbcs rN_HI, rM_HI; \ + mov rD_LO, rM_LO; \ + mov rD_HI, rM_HI; \ + itt cmp_kind; \ + mov##cmp_kind rD_LO, rN_LO; \ + mov##cmp_kind rD_HI, rN_HI diff --git a/lib/builtins/arm/sync_fetch_and_add_4.S b/lib/builtins/arm/sync_fetch_and_add_4.S index 7877d6c46..0d55975b7 100644 --- a/lib/builtins/arm/sync_fetch_and_add_4.S +++ b/lib/builtins/arm/sync_fetch_and_add_4.S @@ -1,20 +1,19 @@ -/*===-- sync_fetch_and_add_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_add_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_add_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_add_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" -/* "adds" is 2 bytes shorter than "add". */ +// "adds" is 2 bytes shorter than "add". #define add_4(rD, rN, rM) add rD, rN, rM SYNC_OP_4(add_4) diff --git a/lib/builtins/arm/sync_fetch_and_add_8.S b/lib/builtins/arm/sync_fetch_and_add_8.S index 1df07a342..18bdd875b 100644 --- a/lib/builtins/arm/sync_fetch_and_add_8.S +++ b/lib/builtins/arm/sync_fetch_and_add_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_add_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_add_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_add_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_add_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_and_4.S b/lib/builtins/arm/sync_fetch_and_and_4.S index 720ff0227..3a76acca6 100644 --- a/lib/builtins/arm/sync_fetch_and_and_4.S +++ b/lib/builtins/arm/sync_fetch_and_and_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_and_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_and_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_and_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_and_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_and_8.S b/lib/builtins/arm/sync_fetch_and_and_8.S index 4f7b5ca7a..3716eff80 100644 --- a/lib/builtins/arm/sync_fetch_and_and_8.S +++ b/lib/builtins/arm/sync_fetch_and_and_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_and_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_and_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_and_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_and_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_max_4.S b/lib/builtins/arm/sync_fetch_and_max_4.S index 43da9c7d4..b9cee4521 100644 --- a/lib/builtins/arm/sync_fetch_and_max_4.S +++ b/lib/builtins/arm/sync_fetch_and_max_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_max_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_max_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_max_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_max_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_max_8.S b/lib/builtins/arm/sync_fetch_and_max_8.S index 898fc6202..06115ab55 100644 --- a/lib/builtins/arm/sync_fetch_and_max_8.S +++ b/lib/builtins/arm/sync_fetch_and_max_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_max_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_max_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_max_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_max_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_min_4.S b/lib/builtins/arm/sync_fetch_and_min_4.S index bba31a03a..60d435a0b 100644 --- a/lib/builtins/arm/sync_fetch_and_min_4.S +++ b/lib/builtins/arm/sync_fetch_and_min_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_min_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_min_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_min_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_min_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_min_8.S b/lib/builtins/arm/sync_fetch_and_min_8.S index e7ccf9fb6..4f3e299d9 100644 --- a/lib/builtins/arm/sync_fetch_and_min_8.S +++ b/lib/builtins/arm/sync_fetch_and_min_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_min_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_min_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_min_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_min_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_nand_4.S b/lib/builtins/arm/sync_fetch_and_nand_4.S index c13dd3945..5a04be0f6 100644 --- a/lib/builtins/arm/sync_fetch_and_nand_4.S +++ b/lib/builtins/arm/sync_fetch_and_nand_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_nand_4.S - -----------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_nand_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_nand_4.S - -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_nand_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_nand_8.S b/lib/builtins/arm/sync_fetch_and_nand_8.S index e8107ab3a..425c94474 100644 --- a/lib/builtins/arm/sync_fetch_and_nand_8.S +++ b/lib/builtins/arm/sync_fetch_and_nand_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_nand_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_nand_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_nand_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_nand_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_or_4.S b/lib/builtins/arm/sync_fetch_and_or_4.S index 6726571a9..f44751b9a 100644 --- a/lib/builtins/arm/sync_fetch_and_or_4.S +++ b/lib/builtins/arm/sync_fetch_and_or_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_or_4.S - -------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_or_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_or_4.S - -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_or_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_or_8.S b/lib/builtins/arm/sync_fetch_and_or_8.S index f7f162c7c..4f18dcf84 100644 --- a/lib/builtins/arm/sync_fetch_and_or_8.S +++ b/lib/builtins/arm/sync_fetch_and_or_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_or_8.S - -------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_or_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_or_8.S - -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_or_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_sub_4.S b/lib/builtins/arm/sync_fetch_and_sub_4.S index b9326b14c..999d48c28 100644 --- a/lib/builtins/arm/sync_fetch_and_sub_4.S +++ b/lib/builtins/arm/sync_fetch_and_sub_4.S @@ -1,20 +1,19 @@ -/*===-- sync_fetch_and_sub_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_sub_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_sub_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_sub_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" -/* "subs" is 2 bytes shorter than "sub". */ +// "subs" is 2 bytes shorter than "sub". #define sub_4(rD, rN, rM) sub rD, rN, rM SYNC_OP_4(sub_4) diff --git a/lib/builtins/arm/sync_fetch_and_sub_8.S b/lib/builtins/arm/sync_fetch_and_sub_8.S index 6ce743e5e..25a4a1076 100644 --- a/lib/builtins/arm/sync_fetch_and_sub_8.S +++ b/lib/builtins/arm/sync_fetch_and_sub_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_sub_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_sub_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_sub_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_sub_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umax_4.S b/lib/builtins/arm/sync_fetch_and_umax_4.S index b8d19ff35..a7b233b15 100644 --- a/lib/builtins/arm/sync_fetch_and_umax_4.S +++ b/lib/builtins/arm/sync_fetch_and_umax_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umax_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umax_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umax_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umax_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umax_8.S b/lib/builtins/arm/sync_fetch_and_umax_8.S index 34442fd77..aa5213ff1 100644 --- a/lib/builtins/arm/sync_fetch_and_umax_8.S +++ b/lib/builtins/arm/sync_fetch_and_umax_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umax_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umax_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umax_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umax_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umin_4.S b/lib/builtins/arm/sync_fetch_and_umin_4.S index 0998e3e10..c7a9c89df 100644 --- a/lib/builtins/arm/sync_fetch_and_umin_4.S +++ b/lib/builtins/arm/sync_fetch_and_umin_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umin_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umin_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umin_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umin_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umin_8.S b/lib/builtins/arm/sync_fetch_and_umin_8.S index 558f91390..8b40541ab 100644 --- a/lib/builtins/arm/sync_fetch_and_umin_8.S +++ b/lib/builtins/arm/sync_fetch_and_umin_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umin_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umin_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umin_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umin_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_xor_4.S b/lib/builtins/arm/sync_fetch_and_xor_4.S index 824f49146..f509191d0 100644 --- a/lib/builtins/arm/sync_fetch_and_xor_4.S +++ b/lib/builtins/arm/sync_fetch_and_xor_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_xor_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_xor_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_xor_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_xor_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_xor_8.S b/lib/builtins/arm/sync_fetch_and_xor_8.S index 073fb9c20..7436eb1d4 100644 --- a/lib/builtins/arm/sync_fetch_and_xor_8.S +++ b/lib/builtins/arm/sync_fetch_and_xor_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_xor_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_xor_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_xor_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_xor_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_synchronize.S b/lib/builtins/arm/sync_synchronize.S index 61d1db910..dd06e7191 100644 --- a/lib/builtins/arm/sync_synchronize.S +++ b/lib/builtins/arm/sync_synchronize.S @@ -1,20 +1,17 @@ //===-- sync_synchronize - Implement memory barrier * ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // When compiling a use of the gcc built-in __sync_synchronize() in thumb1 mode -// the compiler may emit a call to __sync_synchronize. -// On Darwin the implementation jumps to an OS supplied function named +// the compiler may emit a call to __sync_synchronize. +// On Darwin the implementation jumps to an OS supplied function named // OSMemoryBarrier -// .text .syntax unified @@ -31,7 +28,7 @@ END_COMPILERRT_FUNCTION(__sync_synchronize) // tell linker it can break up file at label boundaries .subsections_via_symbols - + #endif NO_EXEC_STACK_DIRECTIVE diff --git a/lib/builtins/arm/truncdfsf2vfp.S b/lib/builtins/arm/truncdfsf2vfp.S index 682e54d3d..a3c0a7346 100644 --- a/lib/builtins/arm/truncdfsf2vfp.S +++ b/lib/builtins/arm/truncdfsf2vfp.S @@ -1,9 +1,8 @@ //===-- truncdfsf2vfp.S - Implement truncdfsf2vfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __truncdfsf2vfp(double a); // // Converts double precision float to signle precision result. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in a R0/R1 pair and a signle precision result is returned in R0. // .syntax unified diff --git a/lib/builtins/arm/udivmodsi4.S b/lib/builtins/arm/udivmodsi4.S index ee3950c9b..0f40575fe 100644 --- a/lib/builtins/arm/udivmodsi4.S +++ b/lib/builtins/arm/udivmodsi4.S @@ -1,16 +1,15 @@ -/*===-- udivmodsi4.S - 32-bit unsigned integer divide and modulus ---------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __udivmodsi4 (32-bit unsigned integer divide and - * modulus) function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- udivmodsi4.S - 32-bit unsigned integer divide and modulus ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __udivmodsi4 (32-bit unsigned integer divide and +// modulus) function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -39,26 +38,25 @@ DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) beq LOCAL_LABEL(divby1) cmp r0, r1 bcc LOCAL_LABEL(quotient0) - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 12 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 12 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # ifdef __ARM_FEATURE_CLZ clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -99,11 +97,11 @@ DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) movhs r4, r3 subhs ip, ip, #(2 * 12) - /* Last block, no need to update r3 or r4. */ + // Last block, no need to update r3 or r4. cmp r1, r4, lsr #1 subls ip, ip, #(1 * 12) - ldr r4, [sp], #8 /* restore r4, we are done with it. */ + ldr r4, [sp], #8 // restore r4, we are done with it. mov r3, #0 JMP(ip) @@ -164,7 +162,7 @@ LOCAL_LABEL(divby1): mov r3, #0 str r3, [r2] JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ LOCAL_LABEL(divby0): mov r0, #0 diff --git a/lib/builtins/arm/udivsi3.S b/lib/builtins/arm/udivsi3.S index 6dea27d40..9b1b035b3 100644 --- a/lib/builtins/arm/udivsi3.S +++ b/lib/builtins/arm/udivsi3.S @@ -1,16 +1,15 @@ -/*===-- udivsi3.S - 32-bit unsigned integer divide ------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __udivsi3 (32-bit unsigned integer divide) - * function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- udivsi3.S - 32-bit unsigned integer divide ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __udivsi3 (32-bit unsigned integer divide) +// function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -40,7 +39,7 @@ LOCAL_LABEL(divby0): JMP(lr) # endif -#else /* ! __ARM_ARCH_EXT_IDIV__ */ +#else // ! __ARM_ARCH_EXT_IDIV__ cmp r1, #1 bcc LOCAL_LABEL(divby0) #if defined(USE_THUMB_1) @@ -63,26 +62,24 @@ LOCAL_LABEL(num_ge_denom): JMPc(lr, cc) #endif - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 12 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 12 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # if defined(__ARM_FEATURE_CLZ) clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -94,7 +91,7 @@ LOCAL_LABEL(num_ge_denom): sub ip, ip, r3, lsl #3 mov r3, #0 bx ip -# else /* No CLZ Feature */ +# else // No CLZ Feature # if defined(USE_THUMB_2) # error THUMB mode requires CLZ or UDIV # endif @@ -160,7 +157,7 @@ LOCAL_LABEL(skip_2): subhs ip, ip, #(2 * BLOCK_SIZE) # endif - /* Last block, no need to update r2 or r3. */ + // Last block, no need to update r2 or r3. # if defined(USE_THUMB_1) lsrs r3, r2, #1 cmp r3, r1 @@ -180,12 +177,12 @@ LOCAL_LABEL(skip_1): JMP(ip) # endif -# endif /* __ARM_FEATURE_CLZ */ +# endif // __ARM_FEATURE_CLZ #define IMM # - /* due to the range limit of branch in Thumb1, we have to place the - block closer */ + // due to the range limit of branch in Thumb1, we have to place the + // block closer LOCAL_LABEL(divby0): movs r0, #0 # if defined(__ARM_EABI__) @@ -204,13 +201,13 @@ LOCAL_LABEL(divby0): blo LOCAL_LABEL(block_skip_##shift); \ subs r0, r0, r2; \ LOCAL_LABEL(block_skip_##shift) :; \ - adcs r3, r3 /* same as ((r3 << 1) | Carry). Carry is set if r0 >= r2. */ + adcs r3, r3 // same as ((r3 << 1) | Carry). Carry is set if r0 >= r2. - /* TODO: if current location counter is not not word aligned, we don't - need the .p2align and nop */ - /* Label div0block must be word-aligned. First align block 31 */ + // TODO: if current location counter is not not word aligned, we don't + // need the .p2align and nop + // Label div0block must be word-aligned. First align block 31 .p2align 2 - nop /* Padding to align div0block as 31 blocks = 310 bytes */ + nop // Padding to align div0block as 31 blocks = 310 bytes #else #define block(shift) \ @@ -256,7 +253,7 @@ LOCAL_LABEL(div0block): mov r0, r3 JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ END_COMPILERRT_FUNCTION(__udivsi3) diff --git a/lib/builtins/arm/umodsi3.S b/lib/builtins/arm/umodsi3.S index 069fad34c..5ab78de17 100644 --- a/lib/builtins/arm/umodsi3.S +++ b/lib/builtins/arm/umodsi3.S @@ -1,16 +1,15 @@ -/*===-- umodsi3.S - 32-bit unsigned integer modulus -----------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __umodsi3 (32-bit unsigned integer modulus) - * function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- umodsi3.S - 32-bit unsigned integer modulus -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __umodsi3 (32-bit unsigned integer modulus) +// function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -38,26 +37,25 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) cmp r0, r1 IT(cc) JMPc(lr, cc) - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 8 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 8 Bytes, for THUMB mode 10 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 8 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 8 Bytes, for THUMB mode 10 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # ifdef __ARM_FEATURE_CLZ clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -94,7 +92,7 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) movhs r2, r3 subhs ip, ip, #(2 * 8) - /* Last block, no need to update r2 or r3. */ + // Last block, no need to update r2 or r3. cmp r1, r2, lsr #1 subls ip, ip, #(1 * 8) @@ -142,7 +140,7 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) LOCAL_LABEL(div0block): block(0) JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ LOCAL_LABEL(divby0): mov r0, #0 diff --git a/lib/builtins/arm/unorddf2vfp.S b/lib/builtins/arm/unorddf2vfp.S index 6625fa8a3..ea36a1cb5 100644 --- a/lib/builtins/arm/unorddf2vfp.S +++ b/lib/builtins/arm/unorddf2vfp.S @@ -1,9 +1,8 @@ //===-- unorddf2vfp.S - Implement unorddf2vfp ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __unorddf2vfp(double a, double b); // // Returns one iff a or b is NaN -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/unordsf2vfp.S b/lib/builtins/arm/unordsf2vfp.S index 0b5da2ba3..731129732 100644 --- a/lib/builtins/arm/unordsf2vfp.S +++ b/lib/builtins/arm/unordsf2vfp.S @@ -1,9 +1,8 @@ //===-- unordsf2vfp.S - Implement unordsf2vfp -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __unordsf2vfp(float a, float b); // // Returns one iff a or b is NaN -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/ashldi3.c b/lib/builtins/ashldi3.c index a5c183600..7c81057a2 100644 --- a/lib/builtins/ashldi3.c +++ b/lib/builtins/ashldi3.c @@ -1,45 +1,38 @@ -/* ====-- ashldi3.c - Implement __ashldi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashldi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +// ====-- ashldi3.c - Implement __ashldi3 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashldi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a << b */ +// Returns: a << b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__ashldi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - dwords input; - dwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - result.s.low = 0; - result.s.high = input.s.low << (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.low = input.s.low << b; - result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_word - b)); - } - return result.all; +COMPILER_RT_ABI di_int __ashldi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + dwords input; + dwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + result.s.low = 0; + result.s.high = input.s.low << (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.low = input.s.low << b; + result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_word - b)); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) COMPILER_RT_ALIAS(__ashldi3); +COMPILER_RT_ALIAS(__ashldi3, __aeabi_llsl) #endif diff --git a/lib/builtins/ashlti3.c b/lib/builtins/ashlti3.c index 638ae845f..2d7bd4a89 100644 --- a/lib/builtins/ashlti3.c +++ b/lib/builtins/ashlti3.c @@ -1,45 +1,38 @@ -/* ===-- ashlti3.c - Implement __ashlti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashlti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashlti3.c - Implement __ashlti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashlti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a << b */ +// Returns: a << b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__ashlti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - twords input; - twords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - result.s.low = 0; - result.s.high = input.s.low << (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.low = input.s.low << b; - result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_dword - b)); - } - return result.all; +COMPILER_RT_ABI ti_int __ashlti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + twords input; + twords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + result.s.low = 0; + result.s.high = input.s.low << (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.low = input.s.low << b; + result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_dword - b)); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/ashrdi3.c b/lib/builtins/ashrdi3.c index 84619965e..b99391322 100644 --- a/lib/builtins/ashrdi3.c +++ b/lib/builtins/ashrdi3.c @@ -1,46 +1,39 @@ -/*===-- ashrdi3.c - Implement __ashrdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashrdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashrdi3.c - Implement __ashrdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashrdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: arithmetic a >> b */ +// Returns: arithmetic a >> b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__ashrdi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - dwords input; - dwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - /* result.s.high = input.s.high < 0 ? -1 : 0 */ - result.s.high = input.s.high >> (bits_in_word - 1); - result.s.low = input.s.high >> (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI di_int __ashrdi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + dwords input; + dwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + // result.s.high = input.s.high < 0 ? -1 : 0 + result.s.high = input.s.high >> (bits_in_word - 1); + result.s.low = input.s.high >> (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) COMPILER_RT_ALIAS(__ashrdi3); +COMPILER_RT_ALIAS(__ashrdi3, __aeabi_lasr) #endif diff --git a/lib/builtins/ashrti3.c b/lib/builtins/ashrti3.c index f78205d96..f573b6d6c 100644 --- a/lib/builtins/ashrti3.c +++ b/lib/builtins/ashrti3.c @@ -1,46 +1,39 @@ -/* ===-- ashrti3.c - Implement __ashrti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashrti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashrti3.c - Implement __ashrti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashrti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: arithmetic a >> b */ +// Returns: arithmetic a >> b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__ashrti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - twords input; - twords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - /* result.s.high = input.s.high < 0 ? -1 : 0 */ - result.s.high = input.s.high >> (bits_in_dword - 1); - result.s.low = input.s.high >> (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI ti_int __ashrti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + twords input; + twords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + // result.s.high = input.s.high < 0 ? -1 : 0 + result.s.high = input.s.high >> (bits_in_dword - 1); + result.s.low = input.s.high >> (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/assembly.h b/lib/builtins/assembly.h index 3f5e59b25..f437cb87f 100644 --- a/lib/builtins/assembly.h +++ b/lib/builtins/assembly.h @@ -1,17 +1,15 @@ -/* ===-- assembly.h - compiler-rt assembler support macros -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file defines macros for use in compiler-rt assembler source. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- assembly.h - compiler-rt assembler support macros -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines macros for use in compiler-rt assembler source. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef COMPILERRT_ASSEMBLY_H #define COMPILERRT_ASSEMBLY_H @@ -69,11 +67,9 @@ #if defined(__arm__) -/* - * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: - * - for '-mthumb -march=armv6' compiler defines '__thumb__' - * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' - */ +// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: +// - for '-mthumb -march=armv6' compiler defines '__thumb__' +// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' #if defined(__thumb2__) || defined(__thumb__) #define DEFINE_CODE_STATE .thumb SEPARATOR #define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR @@ -201,4 +197,4 @@ #define END_COMPILERRT_FUNCTION(name) #endif -#endif /* COMPILERRT_ASSEMBLY_H */ +#endif // COMPILERRT_ASSEMBLY_H diff --git a/lib/builtins/atomic.c b/lib/builtins/atomic.c index ee35e342e..0f82803a6 100644 --- a/lib/builtins/atomic.c +++ b/lib/builtins/atomic.c @@ -1,29 +1,27 @@ -/*===-- atomic.c - Implement support functions for atomic operations.------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * atomic.c defines a set of functions for performing atomic accesses on - * arbitrary-sized memory locations. This design uses locks that should - * be fast in the uncontended case, for two reasons: - * - * 1) This code must work with C programs that do not link to anything - * (including pthreads) and so it should not depend on any pthread - * functions. - * 2) Atomic operations, rather than explicit mutexes, are most commonly used - * on code where contended operations are rate. - * - * To avoid needing a per-object lock, this code allocates an array of - * locks and hashes the object pointers to find the one that it should use. - * For operations that must be atomic on two locations, the lower lock is - * always acquired first, to avoid deadlock. - * - *===----------------------------------------------------------------------=== - */ +//===-- atomic.c - Implement support functions for atomic operations.------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// atomic.c defines a set of functions for performing atomic accesses on +// arbitrary-sized memory locations. This design uses locks that should +// be fast in the uncontended case, for two reasons: +// +// 1) This code must work with C programs that do not link to anything +// (including pthreads) and so it should not depend on any pthread +// functions. +// 2) Atomic operations, rather than explicit mutexes, are most commonly used +// on code where contended operations are rate. +// +// To avoid needing a per-object lock, this code allocates an array of +// locks and hashes the object pointers to find the one that it should use. +// For operations that must be atomic on two locations, the lower lock is +// always acquired first, to avoid deadlock. +// +//===----------------------------------------------------------------------===// #include <stdint.h> #include <string.h> @@ -35,13 +33,14 @@ #pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load) #pragma redefine_extname __atomic_store_c SYMBOL_NAME(__atomic_store) #pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange) -#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME(__atomic_compare_exchange) +#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \ + __atomic_compare_exchange) /// Number of locks. This allocates one page on 32-bit platforms, two on /// 64-bit. This can be specified externally if a different trade between /// memory usage and contention probability is required for a given platform. #ifndef SPINLOCK_COUNT -#define SPINLOCK_COUNT (1<<10) +#define SPINLOCK_COUNT (1 << 10) #endif static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; @@ -52,38 +51,35 @@ static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; //////////////////////////////////////////////////////////////////////////////// #ifdef __FreeBSD__ #include <errno.h> -#include <sys/types.h> #include <machine/atomic.h> +#include <sys/types.h> #include <sys/umtx.h> typedef struct _usem Lock; __inline static void unlock(Lock *l) { - __c11_atomic_store((_Atomic(uint32_t)*)&l->_count, 1, __ATOMIC_RELEASE); + __c11_atomic_store((_Atomic(uint32_t) *)&l->_count, 1, __ATOMIC_RELEASE); __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); if (l->_has_waiters) - _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); + _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); } __inline static void lock(Lock *l) { uint32_t old = 1; - while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t)*)&l->_count, &old, - 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t) *)&l->_count, + &old, 0, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) { _umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0); old = 1; } } /// locks for atomic operations -static Lock locks[SPINLOCK_COUNT] = { [0 ... SPINLOCK_COUNT-1] = {0,1,0} }; +static Lock locks[SPINLOCK_COUNT] = {[0 ... SPINLOCK_COUNT - 1] = {0, 1, 0}}; #elif defined(__APPLE__) #include <libkern/OSAtomic.h> typedef OSSpinLock Lock; -__inline static void unlock(Lock *l) { - OSSpinLockUnlock(l); -} +__inline static void unlock(Lock *l) { OSSpinLockUnlock(l); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. -__inline static void lock(Lock *l) { - OSSpinLockLock(l); -} +__inline static void lock(Lock *l) { OSSpinLockLock(l); } static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 #else @@ -97,20 +93,19 @@ __inline static void unlock(Lock *l) { __inline static void lock(Lock *l) { uintptr_t old = 0; while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)) + __ATOMIC_RELAXED)) old = 0; } /// locks for atomic operations static Lock locks[SPINLOCK_COUNT]; #endif - -/// Returns a lock to use for a given pointer. +/// Returns a lock to use for a given pointer. static __inline Lock *lock_for_pointer(void *ptr) { intptr_t hash = (intptr_t)ptr; // Disregard the lowest 4 bits. We want all values that may be part of the // same memory operation to hash to the same value and therefore use the same - // lock. + // lock. hash >>= 4; // Use the next bits as the basis for the hash intptr_t low = hash & SPINLOCK_MASK; @@ -133,36 +128,44 @@ static __inline Lock *lock_for_pointer(void *ptr) { /// Macro that calls the compiler-generated lock-free versions of functions /// when they exist. -#define LOCK_FREE_CASES() \ - do {\ - switch (size) {\ - case 2:\ - if (IS_LOCK_FREE_2) {\ - LOCK_FREE_ACTION(uint16_t);\ - }\ - case 4:\ - if (IS_LOCK_FREE_4) {\ - LOCK_FREE_ACTION(uint32_t);\ - }\ - case 8:\ - if (IS_LOCK_FREE_8) {\ - LOCK_FREE_ACTION(uint64_t);\ - }\ - case 16:\ - if (IS_LOCK_FREE_16) {\ - /* FIXME: __uint128_t isn't available on 32 bit platforms. - LOCK_FREE_ACTION(__uint128_t);*/\ - }\ - }\ +#define LOCK_FREE_CASES() \ + do { \ + switch (size) { \ + case 1: \ + if (IS_LOCK_FREE_1) { \ + LOCK_FREE_ACTION(uint8_t); \ + } \ + break; \ + case 2: \ + if (IS_LOCK_FREE_2) { \ + LOCK_FREE_ACTION(uint16_t); \ + } \ + break; \ + case 4: \ + if (IS_LOCK_FREE_4) { \ + LOCK_FREE_ACTION(uint32_t); \ + } \ + break; \ + case 8: \ + if (IS_LOCK_FREE_8) { \ + LOCK_FREE_ACTION(uint64_t); \ + } \ + break; \ + case 16: \ + if (IS_LOCK_FREE_16) { \ + /* FIXME: __uint128_t isn't available on 32 bit platforms. \ + LOCK_FREE_ACTION(__uint128_t);*/ \ + } \ + break; \ + } \ } while (0) - /// An atomic load operation. This is atomic with respect to the source /// pointer only. void __atomic_load_c(int size, void *src, void *dest, int model) { -#define LOCK_FREE_ACTION(type) \ - *((type*)dest) = __c11_atomic_load((_Atomic(type)*)src, model);\ - return; +#define LOCK_FREE_ACTION(type) \ + *((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(src); @@ -174,9 +177,9 @@ void __atomic_load_c(int size, void *src, void *dest, int model) { /// An atomic store operation. This is atomic with respect to the destination /// pointer only. void __atomic_store_c(int size, void *dest, void *src, int model) { -#define LOCK_FREE_ACTION(type) \ - __c11_atomic_store((_Atomic(type)*)dest, *(type*)dest, model);\ - return; +#define LOCK_FREE_ACTION(type) \ + __c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(dest); @@ -189,12 +192,13 @@ void __atomic_store_c(int size, void *dest, void *src, int model) { /// to the value at *expected, then this copies value at *desired to *ptr. If /// they are not, then this stores the current value from *ptr in *expected. /// -/// This function returns 1 if the exchange takes place or 0 if it fails. +/// This function returns 1 if the exchange takes place or 0 if it fails. int __atomic_compare_exchange_c(int size, void *ptr, void *expected, - void *desired, int success, int failure) { -#define LOCK_FREE_ACTION(type) \ - return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, (type*)expected,\ - *(type*)desired, success, failure) + void *desired, int success, int failure) { +#define LOCK_FREE_ACTION(type) \ + return __c11_atomic_compare_exchange_strong( \ + (_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \ + failure) LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); @@ -212,10 +216,10 @@ int __atomic_compare_exchange_c(int size, void *ptr, void *expected, /// Performs an atomic exchange operation between two pointers. This is atomic /// with respect to the target address. void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { -#define LOCK_FREE_ACTION(type) \ - *(type*)old = __c11_atomic_exchange((_Atomic(type)*)ptr, *(type*)val,\ - model);\ - return; +#define LOCK_FREE_ACTION(type) \ + *(type *)old = \ + __c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); @@ -230,96 +234,96 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { // specialised versions of the above functions. //////////////////////////////////////////////////////////////////////////////// #ifdef __SIZEOF_INT128__ -#define OPTIMISED_CASES\ - OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\ - OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\ - OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\ - OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t)\ +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ + OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) \ OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t) #else -#define OPTIMISED_CASES\ - OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\ - OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\ - OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\ +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) #endif -#define OPTIMISED_CASE(n, lockfree, type)\ -type __atomic_load_##n(type *src, int model) {\ - if (lockfree)\ - return __c11_atomic_load((_Atomic(type)*)src, model);\ - Lock *l = lock_for_pointer(src);\ - lock(l);\ - type val = *src;\ - unlock(l);\ - return val;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_load_##n(type *src, int model) { \ + if (lockfree) \ + return __c11_atomic_load((_Atomic(type) *)src, model); \ + Lock *l = lock_for_pointer(src); \ + lock(l); \ + type val = *src; \ + unlock(l); \ + return val; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -void __atomic_store_##n(type *dest, type val, int model) {\ - if (lockfree) {\ - __c11_atomic_store((_Atomic(type)*)dest, val, model);\ - return;\ - }\ - Lock *l = lock_for_pointer(dest);\ - lock(l);\ - *dest = val;\ - unlock(l);\ - return;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + void __atomic_store_##n(type *dest, type val, int model) { \ + if (lockfree) { \ + __c11_atomic_store((_Atomic(type) *)dest, val, model); \ + return; \ + } \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + *dest = val; \ + unlock(l); \ + return; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -type __atomic_exchange_##n(type *dest, type val, int model) {\ - if (lockfree)\ - return __c11_atomic_exchange((_Atomic(type)*)dest, val, model);\ - Lock *l = lock_for_pointer(dest);\ - lock(l);\ - type tmp = *dest;\ - *dest = val;\ - unlock(l);\ - return tmp;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_exchange_##n(type *dest, type val, int model) { \ + if (lockfree) \ + return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + type tmp = *dest; \ + *dest = val; \ + unlock(l); \ + return tmp; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired,\ - int success, int failure) {\ - if (lockfree)\ - return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, expected, desired,\ - success, failure);\ - Lock *l = lock_for_pointer(ptr);\ - lock(l);\ - if (*ptr == *expected) {\ - *ptr = desired;\ - unlock(l);\ - return 1;\ - }\ - *expected = *ptr;\ - unlock(l);\ - return 0;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ + int success, int failure) { \ + if (lockfree) \ + return __c11_atomic_compare_exchange_strong( \ + (_Atomic(type) *)ptr, expected, desired, success, failure); \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + if (*ptr == *expected) { \ + *ptr = desired; \ + unlock(l); \ + return 1; \ + } \ + *expected = *ptr; \ + unlock(l); \ + return 0; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE //////////////////////////////////////////////////////////////////////////////// // Atomic read-modify-write operations for integers of various sizes. //////////////////////////////////////////////////////////////////////////////// -#define ATOMIC_RMW(n, lockfree, type, opname, op) \ -type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) {\ - if (lockfree) \ - return __c11_atomic_fetch_##opname((_Atomic(type)*)ptr, val, model);\ - Lock *l = lock_for_pointer(ptr);\ - lock(l);\ - type tmp = *ptr;\ - *ptr = tmp op val;\ - unlock(l);\ - return tmp;\ -} +#define ATOMIC_RMW(n, lockfree, type, opname, op) \ + type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \ + if (lockfree) \ + return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + type tmp = *ptr; \ + *ptr = tmp op val; \ + unlock(l); \ + return tmp; \ + } #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) OPTIMISED_CASES diff --git a/lib/builtins/atomic_flag_clear.c b/lib/builtins/atomic_flag_clear.c index da912af64..983e5d7f0 100644 --- a/lib/builtins/atomic_flag_clear.c +++ b/lib/builtins/atomic_flag_clear.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_clear.c -------------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_clear from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_clear.c -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_clear from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_clear_explicit.c b/lib/builtins/atomic_flag_clear_explicit.c index 1059b787f..e61c06476 100644 --- a/lib/builtins/atomic_flag_clear_explicit.c +++ b/lib/builtins/atomic_flag_clear_explicit.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_clear_explicit.c ----------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_clear_explicit from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_clear_explicit.c --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_clear_explicit from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_test_and_set.c b/lib/builtins/atomic_flag_test_and_set.c index e8811d39e..ee22b08b5 100644 --- a/lib/builtins/atomic_flag_test_and_set.c +++ b/lib/builtins/atomic_flag_test_and_set.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_test_and_set.c ------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_test_and_set from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_test_and_set.c ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_test_and_set from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_test_and_set_explicit.c b/lib/builtins/atomic_flag_test_and_set_explicit.c index 5c8c2df90..8c9d03994 100644 --- a/lib/builtins/atomic_flag_test_and_set_explicit.c +++ b/lib/builtins/atomic_flag_test_and_set_explicit.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_test_and_set_explicit.c ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_test_and_set_explicit from C11's stdatomic.h - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_test_and_set_explicit.c -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_test_and_set_explicit from C11's stdatomic.h +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_signal_fence.c b/lib/builtins/atomic_signal_fence.c index 9ccc2ae60..f4f5169d3 100644 --- a/lib/builtins/atomic_signal_fence.c +++ b/lib/builtins/atomic_signal_fence.c @@ -1,16 +1,14 @@ -/*===-- atomic_signal_fence.c -----------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_signal_fence from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_signal_fence.c ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_signal_fence from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_thread_fence.c b/lib/builtins/atomic_thread_fence.c index d22560151..5659ecb0b 100644 --- a/lib/builtins/atomic_thread_fence.c +++ b/lib/builtins/atomic_thread_fence.c @@ -1,16 +1,14 @@ -/*===-- atomic_thread_fence.c -----------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_thread_fence from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_thread_fence.c ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_thread_fence from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/bswapdi2.c b/lib/builtins/bswapdi2.c index eb220007b..cd049f58e 100644 --- a/lib/builtins/bswapdi2.c +++ b/lib/builtins/bswapdi2.c @@ -1,16 +1,14 @@ -/* ===-- bswapdi2.c - Implement __bswapdi2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __bswapdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- bswapdi2.c - Implement __bswapdi2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __bswapdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" diff --git a/lib/builtins/bswapsi2.c b/lib/builtins/bswapsi2.c index 5d941e69f..ec566d61a 100644 --- a/lib/builtins/bswapsi2.c +++ b/lib/builtins/bswapsi2.c @@ -1,23 +1,20 @@ -/* ===-- bswapsi2.c - Implement __bswapsi2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __bswapsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- bswapsi2.c - Implement __bswapsi2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __bswapsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" COMPILER_RT_ABI uint32_t __bswapsi2(uint32_t u) { - return ( - (((u)&0xff000000) >> 24) | - (((u)&0x00ff0000) >> 8) | - (((u)&0x0000ff00) << 8) | - (((u)&0x000000ff) << 24)); + return ((((u)&0xff000000) >> 24) | + (((u)&0x00ff0000) >> 8) | + (((u)&0x0000ff00) << 8) | + (((u)&0x000000ff) << 24)); } diff --git a/lib/builtins/clear_cache.c b/lib/builtins/clear_cache.c index 9dcab344a..76dc1968c 100644 --- a/lib/builtins/clear_cache.c +++ b/lib/builtins/clear_cache.c @@ -1,179 +1,164 @@ -/* ===-- clear_cache.c - Implement __clear_cache ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clear_cache.c - Implement __clear_cache ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <assert.h> #include <stddef.h> #if __APPLE__ - #include <libkern/OSCacheControl.h> +#include <libkern/OSCacheControl.h> #endif #if defined(_WIN32) -/* Forward declare Win32 APIs since the GCC mode driver does not handle the - newer SDKs as well as needed. */ +// Forward declare Win32 APIs since the GCC mode driver does not handle the +// newer SDKs as well as needed. uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress, uintptr_t dwSize); uintptr_t GetCurrentProcess(void); #endif #if defined(__FreeBSD__) && defined(__arm__) - #include <sys/types.h> - #include <machine/sysarch.h> +#include <machine/sysarch.h> +#include <sys/types.h> #endif #if defined(__NetBSD__) && defined(__arm__) - #include <machine/sysarch.h> +#include <machine/sysarch.h> #endif #if defined(__OpenBSD__) && defined(__mips__) - #include <sys/types.h> - #include <machine/sysarch.h> +#include <machine/sysarch.h> +#include <sys/types.h> #endif #if defined(__linux__) && defined(__mips__) - #include <sys/cachectl.h> - #include <sys/syscall.h> - #include <unistd.h> - #if defined(__ANDROID__) && defined(__LP64__) - /* - * clear_mips_cache - Invalidates instruction cache for Mips. - */ - static void clear_mips_cache(const void* Addr, size_t Size) { - __asm__ volatile ( - ".set push\n" - ".set noreorder\n" - ".set noat\n" - "beq %[Size], $zero, 20f\n" /* If size == 0, branch around. */ - "nop\n" - "daddu %[Size], %[Addr], %[Size]\n" /* Calculate end address + 1 */ - "rdhwr $v0, $1\n" /* Get step size for SYNCI. - $1 is $HW_SYNCI_Step */ - "beq $v0, $zero, 20f\n" /* If no caches require - synchronization, branch - around. */ - "nop\n" - "10:\n" - "synci 0(%[Addr])\n" /* Synchronize all caches around - address. */ - "daddu %[Addr], %[Addr], $v0\n" /* Add step size. */ - "sltu $at, %[Addr], %[Size]\n" /* Compare current with end - address. */ - "bne $at, $zero, 10b\n" /* Branch if more to do. */ - "nop\n" - "sync\n" /* Clear memory hazards. */ - "20:\n" - "bal 30f\n" - "nop\n" - "30:\n" - "daddiu $ra, $ra, 12\n" /* $ra has a value of $pc here. - Add offset of 12 to point to the - instruction after the last nop. - */ - "jr.hb $ra\n" /* Return, clearing instruction - hazards. */ - "nop\n" - ".set pop\n" - : [Addr] "+r"(Addr), [Size] "+r"(Size) - :: "at", "ra", "v0", "memory" - ); - } - #endif +#include <sys/cachectl.h> +#include <sys/syscall.h> +#include <unistd.h> +#if defined(__ANDROID__) && defined(__LP64__) +// clear_mips_cache - Invalidates instruction cache for Mips. +static void clear_mips_cache(const void *Addr, size_t Size) { + __asm__ volatile( + ".set push\n" + ".set noreorder\n" + ".set noat\n" + "beq %[Size], $zero, 20f\n" // If size == 0, branch around. + "nop\n" + "daddu %[Size], %[Addr], %[Size]\n" // Calculate end address + 1 + "rdhwr $v0, $1\n" // Get step size for SYNCI. + // $1 is $HW_SYNCI_Step + "beq $v0, $zero, 20f\n" // If no caches require + // synchronization, branch + // around. + "nop\n" + "10:\n" + "synci 0(%[Addr])\n" // Synchronize all caches around + // address. + "daddu %[Addr], %[Addr], $v0\n" // Add step size. + "sltu $at, %[Addr], %[Size]\n" // Compare current with end + // address. + "bne $at, $zero, 10b\n" // Branch if more to do. + "nop\n" + "sync\n" // Clear memory hazards. + "20:\n" + "bal 30f\n" + "nop\n" + "30:\n" + "daddiu $ra, $ra, 12\n" // $ra has a value of $pc here. + // Add offset of 12 to point to the + // instruction after the last nop. + // + "jr.hb $ra\n" // Return, clearing instruction + // hazards. + "nop\n" + ".set pop\n" + : [ Addr ] "+r"(Addr), [ Size ] "+r"(Size)::"at", "ra", "v0", "memory"); +} +#endif #endif -/* - * The compiler generates calls to __clear_cache() when creating - * trampoline functions on the stack for use with nested functions. - * It is expected to invalidate the instruction cache for the - * specified range. - */ +// The compiler generates calls to __clear_cache() when creating +// trampoline functions on the stack for use with nested functions. +// It is expected to invalidate the instruction cache for the +// specified range. void __clear_cache(void *start, void *end) { #if __i386__ || __x86_64__ || defined(_M_IX86) || defined(_M_X64) -/* - * Intel processors have a unified instruction and data cache - * so there is nothing to do - */ +// Intel processors have a unified instruction and data cache +// so there is nothing to do #elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__)) - FlushInstructionCache(GetCurrentProcess(), start, end - start); + FlushInstructionCache(GetCurrentProcess(), start, end - start); #elif defined(__arm__) && !defined(__APPLE__) - #if defined(__FreeBSD__) || defined(__NetBSD__) - struct arm_sync_icache_args arg; - - arg.addr = (uintptr_t)start; - arg.len = (uintptr_t)end - (uintptr_t)start; - - sysarch(ARM_SYNC_ICACHE, &arg); - #elif defined(__linux__) - /* - * We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but - * it also brought many other unused defines, as well as a dependency on - * kernel headers to be installed. - * - * This value is stable at least since Linux 3.13 and should remain so for - * compatibility reasons, warranting it's re-definition here. - */ - #define __ARM_NR_cacheflush 0x0f0002 - register int start_reg __asm("r0") = (int) (intptr_t) start; - const register int end_reg __asm("r1") = (int) (intptr_t) end; - const register int flags __asm("r2") = 0; - const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; - __asm __volatile("svc 0x0" - : "=r"(start_reg) - : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), - "r"(flags)); - assert(start_reg == 0 && "Cache flush syscall failed."); - #else - compilerrt_abort(); - #endif +#if defined(__FreeBSD__) || defined(__NetBSD__) + struct arm_sync_icache_args arg; + + arg.addr = (uintptr_t)start; + arg.len = (uintptr_t)end - (uintptr_t)start; + + sysarch(ARM_SYNC_ICACHE, &arg); +#elif defined(__linux__) +// We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but +// it also brought many other unused defines, as well as a dependency on +// kernel headers to be installed. +// +// This value is stable at least since Linux 3.13 and should remain so for +// compatibility reasons, warranting it's re-definition here. +#define __ARM_NR_cacheflush 0x0f0002 + register int start_reg __asm("r0") = (int)(intptr_t)start; + const register int end_reg __asm("r1") = (int)(intptr_t)end; + const register int flags __asm("r2") = 0; + const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; + __asm __volatile("svc 0x0" + : "=r"(start_reg) + : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), "r"(flags)); + assert(start_reg == 0 && "Cache flush syscall failed."); +#else + compilerrt_abort(); +#endif #elif defined(__linux__) && defined(__mips__) - const uintptr_t start_int = (uintptr_t) start; - const uintptr_t end_int = (uintptr_t) end; - #if defined(__ANDROID__) && defined(__LP64__) - // Call synci implementation for short address range. - const uintptr_t address_range_limit = 256; - if ((end_int - start_int) <= address_range_limit) { - clear_mips_cache(start, (end_int - start_int)); - } else { - syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); - } - #else - syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); - #endif + const uintptr_t start_int = (uintptr_t)start; + const uintptr_t end_int = (uintptr_t)end; +#if defined(__ANDROID__) && defined(__LP64__) + // Call synci implementation for short address range. + const uintptr_t address_range_limit = 256; + if ((end_int - start_int) <= address_range_limit) { + clear_mips_cache(start, (end_int - start_int)); + } else { + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); + } +#else + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); +#endif #elif defined(__mips__) && defined(__OpenBSD__) cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE); #elif defined(__aarch64__) && !defined(__APPLE__) - uint64_t xstart = (uint64_t)(uintptr_t) start; - uint64_t xend = (uint64_t)(uintptr_t) end; + uint64_t xstart = (uint64_t)(uintptr_t)start; + uint64_t xend = (uint64_t)(uintptr_t)end; uint64_t addr; // Get Cache Type Info uint64_t ctr_el0; __asm __volatile("mrs %0, ctr_el0" : "=r"(ctr_el0)); - /* - * dc & ic instructions must use 64bit registers so we don't use - * uintptr_t in case this runs in an IPL32 environment. - */ + // dc & ic instructions must use 64bit registers so we don't use + // uintptr_t in case this runs in an IPL32 environment. const size_t dcache_line_size = 4 << ((ctr_el0 >> 16) & 15); for (addr = xstart & ~(dcache_line_size - 1); addr < xend; addr += dcache_line_size) - __asm __volatile("dc cvau, %0" :: "r"(addr)); + __asm __volatile("dc cvau, %0" ::"r"(addr)); __asm __volatile("dsb ish"); const size_t icache_line_size = 4 << ((ctr_el0 >> 0) & 15); for (addr = xstart & ~(icache_line_size - 1); addr < xend; addr += icache_line_size) - __asm __volatile("ic ivau, %0" :: "r"(addr)); + __asm __volatile("ic ivau, %0" ::"r"(addr)); __asm __volatile("isb sy"); -#elif defined (__powerpc64__) +#elif defined(__powerpc64__) const size_t line_size = 32; const size_t len = (uintptr_t)end - (uintptr_t)start; @@ -189,11 +174,11 @@ void __clear_cache(void *start, void *end) { __asm__ volatile("icbi 0, %0" : : "r"(line)); __asm__ volatile("isync"); #else - #if __APPLE__ - /* On Darwin, sys_icache_invalidate() provides this functionality */ - sys_icache_invalidate(start, end-start); - #else - compilerrt_abort(); - #endif +#if __APPLE__ + // On Darwin, sys_icache_invalidate() provides this functionality + sys_icache_invalidate(start, end - start); +#else + compilerrt_abort(); +#endif #endif } diff --git a/lib/builtins/clzdi2.c b/lib/builtins/clzdi2.c index 1819e6be4..a0bacb2ae 100644 --- a/lib/builtins/clzdi2.c +++ b/lib/builtins/clzdi2.c @@ -1,40 +1,35 @@ -/* ===-- clzdi2.c - Implement __clzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzdi2.c - Implement __clzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits #if !defined(__clang__) && \ - ((defined(__sparc__) && defined(__arch64__)) || \ - defined(__mips64) || \ + ((defined(__sparc__) && defined(__arch64__)) || defined(__mips64) || \ (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) -/* On 64-bit architectures with neither a native clz instruction nor a native - * ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than - * __clzsi2, leading to infinite recursion. */ +// On 64-bit architectures with neither a native clz instruction nor a native +// ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than +// __clzsi2, leading to infinite recursion. #define __builtin_clz(a) __clzsi2(a) extern si_int __clzsi2(si_int); #endif -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzdi2(di_int a) -{ - dwords x; - x.all = a; - const si_int f = -(x.s.high == 0); - return __builtin_clz((x.s.high & ~f) | (x.s.low & f)) + - (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __clzdi2(di_int a) { + dwords x; + x.all = a; + const si_int f = -(x.s.high == 0); + return __builtin_clz((x.s.high & ~f) | (x.s.low & f)) + + (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); } diff --git a/lib/builtins/clzsi2.c b/lib/builtins/clzsi2.c index 25b8ed2c4..3f9f27f41 100644 --- a/lib/builtins/clzsi2.c +++ b/lib/builtins/clzsi2.c @@ -1,53 +1,48 @@ -/* ===-- clzsi2.c - Implement __clzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzsi2.c - Implement __clzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzsi2(si_int a) -{ - su_int x = (su_int)a; - si_int t = ((x & 0xFFFF0000) == 0) << 4; /* if (x is small) t = 16 else 0 */ - x >>= 16 - t; /* x = [0 - 0xFFFF] */ - su_int r = t; /* r = [0, 16] */ - /* return r + clz(x) */ - t = ((x & 0xFF00) == 0) << 3; - x >>= 8 - t; /* x = [0 - 0xFF] */ - r += t; /* r = [0, 8, 16, 24] */ - /* return r + clz(x) */ - t = ((x & 0xF0) == 0) << 2; - x >>= 4 - t; /* x = [0 - 0xF] */ - r += t; /* r = [0, 4, 8, 12, 16, 20, 24, 28] */ - /* return r + clz(x) */ - t = ((x & 0xC) == 0) << 1; - x >>= 2 - t; /* x = [0 - 3] */ - r += t; /* r = [0 - 30] and is even */ - /* return r + clz(x) */ -/* switch (x) - * { - * case 0: - * return r + 2; - * case 1: - * return r + 1; - * case 2: - * case 3: - * return r; - * } - */ - return r + ((2 - x) & -((x & 2) == 0)); +COMPILER_RT_ABI si_int __clzsi2(si_int a) { + su_int x = (su_int)a; + si_int t = ((x & 0xFFFF0000) == 0) << 4; // if (x is small) t = 16 else 0 + x >>= 16 - t; // x = [0 - 0xFFFF] + su_int r = t; // r = [0, 16] + // return r + clz(x) + t = ((x & 0xFF00) == 0) << 3; + x >>= 8 - t; // x = [0 - 0xFF] + r += t; // r = [0, 8, 16, 24] + // return r + clz(x) + t = ((x & 0xF0) == 0) << 2; + x >>= 4 - t; // x = [0 - 0xF] + r += t; // r = [0, 4, 8, 12, 16, 20, 24, 28] + // return r + clz(x) + t = ((x & 0xC) == 0) << 1; + x >>= 2 - t; // x = [0 - 3] + r += t; // r = [0 - 30] and is even + // return r + clz(x) + // switch (x) + // { + // case 0: + // return r + 2; + // case 1: + // return r + 1; + // case 2: + // case 3: + // return r; + // } + return r + ((2 - x) & -((x & 2) == 0)); } diff --git a/lib/builtins/clzti2.c b/lib/builtins/clzti2.c index 15a7b3c90..0c787104c 100644 --- a/lib/builtins/clzti2.c +++ b/lib/builtins/clzti2.c @@ -1,33 +1,29 @@ -/* ===-- clzti2.c - Implement __clzti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzti2.c - Implement __clzti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzti2(ti_int a) -{ - twords x; - x.all = a; - const di_int f = -(x.s.high == 0); - return __builtin_clzll((x.s.high & ~f) | (x.s.low & f)) + - ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __clzti2(ti_int a) { + twords x; + x.all = a; + const di_int f = -(x.s.high == 0); + return __builtin_clzll((x.s.high & ~f) | (x.s.low & f)) + + ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/cmpdi2.c b/lib/builtins/cmpdi2.c index 52634d9c3..951db85b5 100644 --- a/lib/builtins/cmpdi2.c +++ b/lib/builtins/cmpdi2.c @@ -1,51 +1,42 @@ -/* ===-- cmpdi2.c - Implement __cmpdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __cmpdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- cmpdi2.c - Implement __cmpdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __cmpdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: if (a < b) returns 0 -* if (a == b) returns 1 -* if (a > b) returns 2 -*/ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__cmpdi2(di_int a, di_int b) -{ - dwords x; - x.all = a; - dwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __cmpdi2(di_int a, di_int b) { + dwords x; + x.all = a; + dwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } #ifdef __ARM_EABI__ -/* Returns: if (a < b) returns -1 -* if (a == b) returns 0 -* if (a > b) returns 1 -*/ -COMPILER_RT_ABI si_int -__aeabi_lcmp(di_int a, di_int b) -{ - return __cmpdi2(a, b) - 1; +// Returns: if (a < b) returns -1 +// if (a == b) returns 0 +// if (a > b) returns 1 +COMPILER_RT_ABI si_int __aeabi_lcmp(di_int a, di_int b) { + return __cmpdi2(a, b) - 1; } #endif - diff --git a/lib/builtins/cmpti2.c b/lib/builtins/cmpti2.c index 2c8b56e29..7f0ee1b51 100644 --- a/lib/builtins/cmpti2.c +++ b/lib/builtins/cmpti2.c @@ -1,42 +1,37 @@ -/* ===-- cmpti2.c - Implement __cmpti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __cmpti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- cmpti2.c - Implement __cmpti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __cmpti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__cmpti2(ti_int a, ti_int b) -{ - twords x; - x.all = a; - twords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __cmpti2(ti_int a, ti_int b) { + twords x; + x.all = a; + twords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/comparedf2.c b/lib/builtins/comparedf2.c index 44e5d2b28..58290d87d 100644 --- a/lib/builtins/comparedf2.c +++ b/lib/builtins/comparedf2.c @@ -1,9 +1,8 @@ //===-- lib/comparedf2.c - Double-precision comparisons -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,79 +39,93 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; + +COMPILER_RT_ABI enum LE_RESULT __ledf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; -COMPILER_RT_ABI enum LE_RESULT -__ledf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a floating-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - else { - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a floating-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } + + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + else { + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmpdf2, __ledf2); +COMPILER_RT_ALIAS(__ledf2, __cmpdf2) #endif +COMPILER_RT_ALIAS(__ledf2, __eqdf2) +COMPILER_RT_ALIAS(__ledf2, __ltdf2) +COMPILER_RT_ALIAS(__ledf2, __nedf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; -COMPILER_RT_ABI enum GE_RESULT -__gedf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } +COMPILER_RT_ABI enum GE_RESULT __gedf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } +COMPILER_RT_ALIAS(__gedf2, __gtdf2) + COMPILER_RT_ABI int __unorddf2(fp_t a, fp_t b) { const rep_t aAbs = toRep(a) & absMask; @@ -120,34 +133,19 @@ __unorddf2(fp_t a, fp_t b) { return aAbs > infRep || bAbs > infRep; } -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT -__eqdf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__ltdf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__nedf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum GE_RESULT -__gtdf2(fp_t a, fp_t b) { - return __gedf2(a, b); -} - #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { - return __unorddf2(a, b); -} +AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { return __unorddf2(a, b); } #else -AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unorddf2); +COMPILER_RT_ALIAS(__unorddf2, __aeabi_dcmpun) #endif #endif + +#if defined(_WIN32) && !defined(__MINGW32__) +// The alias mechanism doesn't work on Windows except for MinGW, so emit +// wrapper functions. +int __eqdf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __ltdf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __nedf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __gtdf2(fp_t a, fp_t b) { return __gedf2(a, b); } +#endif diff --git a/lib/builtins/comparesf2.c b/lib/builtins/comparesf2.c index 43cd6a6a7..1cb99e468 100644 --- a/lib/builtins/comparesf2.c +++ b/lib/builtins/comparesf2.c @@ -1,9 +1,8 @@ //===-- lib/comparesf2.c - Single-precision comparisons -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,79 +39,93 @@ #define SINGLE_PRECISION #include "fp_lib.h" -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; + +COMPILER_RT_ABI enum LE_RESULT __lesf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; -COMPILER_RT_ABI enum LE_RESULT -__lesf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a fp_ting-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - else { - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a fp_ting-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } + + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + else { + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmpsf2, __lesf2); +COMPILER_RT_ALIAS(__lesf2, __cmpsf2) #endif +COMPILER_RT_ALIAS(__lesf2, __eqsf2) +COMPILER_RT_ALIAS(__lesf2, __ltsf2) +COMPILER_RT_ALIAS(__lesf2, __nesf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; -COMPILER_RT_ABI enum GE_RESULT -__gesf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } +COMPILER_RT_ABI enum GE_RESULT __gesf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } +COMPILER_RT_ALIAS(__gesf2, __gtsf2) + COMPILER_RT_ABI int __unordsf2(fp_t a, fp_t b) { const rep_t aAbs = toRep(a) & absMask; @@ -120,34 +133,19 @@ __unordsf2(fp_t a, fp_t b) { return aAbs > infRep || bAbs > infRep; } -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT -__eqsf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__ltsf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__nesf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum GE_RESULT -__gtsf2(fp_t a, fp_t b) { - return __gesf2(a, b); -} - #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { - return __unordsf2(a, b); -} +AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); } #else -AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unordsf2); +COMPILER_RT_ALIAS(__unordsf2, __aeabi_fcmpun) #endif #endif + +#if defined(_WIN32) && !defined(__MINGW32__) +// The alias mechanism doesn't work on Windows except for MinGW, so emit +// wrapper functions. +int __eqsf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __ltsf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __nesf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __gtsf2(fp_t a, fp_t b) { return __gesf2(a, b); } +#endif diff --git a/lib/builtins/comparetf2.c b/lib/builtins/comparetf2.c index c0ad8ed0a..2eb34cf37 100644 --- a/lib/builtins/comparetf2.c +++ b/lib/builtins/comparetf2.c @@ -1,9 +1,8 @@ //===-- lib/comparetf2.c - Quad-precision comparisons -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,98 +40,95 @@ #include "fp_lib.h" #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; COMPILER_RT_ABI enum LE_RESULT __letf2(fp_t a, fp_t b) { - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a floating-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - else { - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; + + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a floating-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } else { + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmptf2, __letf2); +COMPILER_RT_ALIAS(__letf2, __cmptf2) #endif +COMPILER_RT_ALIAS(__letf2, __eqtf2) +COMPILER_RT_ALIAS(__letf2, __lttf2) +COMPILER_RT_ALIAS(__letf2, __netf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; COMPILER_RT_ABI enum GE_RESULT __getf2(fp_t a, fp_t b) { - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } -} - -COMPILER_RT_ABI int __unordtf2(fp_t a, fp_t b) { - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - return aAbs > infRep || bAbs > infRep; -} - -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT __eqtf2(fp_t a, fp_t b) { - return __letf2(a, b); + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } -COMPILER_RT_ABI enum LE_RESULT __lttf2(fp_t a, fp_t b) { - return __letf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT __netf2(fp_t a, fp_t b) { - return __letf2(a, b); -} +COMPILER_RT_ALIAS(__getf2, __gttf2) -COMPILER_RT_ABI enum GE_RESULT __gttf2(fp_t a, fp_t b) { - return __getf2(a, b); +COMPILER_RT_ABI int __unordtf2(fp_t a, fp_t b) { + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + return aAbs > infRep || bAbs > infRep; } #endif diff --git a/lib/builtins/cpu_model.c b/lib/builtins/cpu_model.c index fb2b899fc..fcc80b562 100644 --- a/lib/builtins/cpu_model.c +++ b/lib/builtins/cpu_model.c @@ -1,9 +1,8 @@ //===-- cpu_model.c - Support for __cpu_model builtin ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#if (defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64__) || defined(_M_X64)) && \ +#if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \ + defined(_M_X64)) && \ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) #include <assert.h> @@ -32,8 +31,8 @@ #endif enum VendorSignatures { - SIG_INTEL = 0x756e6547 /* Genu */, - SIG_AMD = 0x68747541 /* Auth */ + SIG_INTEL = 0x756e6547, // Genu + SIG_AMD = 0x68747541, // Auth }; enum ProcessorVendors { @@ -81,6 +80,8 @@ enum ProcessorSubtypes { INTEL_COREI7_CANNONLAKE, INTEL_COREI7_ICELAKE_CLIENT, INTEL_COREI7_ICELAKE_SERVER, + AMDFAM17H_ZNVER2, + INTEL_COREI7_CASCADELAKE, CPU_SUBTYPE_MAX }; @@ -266,10 +267,11 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, } } -static void -getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, - unsigned Brand_id, unsigned Features, - unsigned *Type, unsigned *Subtype) { +static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Brand_id, + unsigned Features, + unsigned Features2, unsigned *Type, + unsigned *Subtype) { if (Brand_id != 0) return; switch (Family) { @@ -295,7 +297,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, case 0x1e: // Intel(R) Core(TM) i7 CPU 870 @ 2.93GHz. // As found in a Summer 2010 model iMac. case 0x1f: - case 0x2e: // Nehalem EX + case 0x2e: // Nehalem EX *Type = INTEL_COREI7; // "nehalem" *Subtype = INTEL_COREI7_NEHALEM; break; @@ -313,7 +315,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Subtype = INTEL_COREI7_SANDYBRIDGE; break; case 0x3a: - case 0x3e: // Ivy Bridge EP + case 0x3e: // Ivy Bridge EP *Type = INTEL_COREI7; // "ivybridge" *Subtype = INTEL_COREI7_IVYBRIDGE; break; @@ -337,10 +339,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, break; // Skylake: - case 0x4e: // Skylake mobile - case 0x5e: // Skylake desktop - case 0x8e: // Kaby Lake mobile - case 0x9e: // Kaby Lake desktop + case 0x4e: // Skylake mobile + case 0x5e: // Skylake desktop + case 0x8e: // Kaby Lake mobile + case 0x9e: // Kaby Lake desktop *Type = INTEL_COREI7; // "skylake" *Subtype = INTEL_COREI7_SKYLAKE; break; @@ -348,7 +350,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, // Skylake Xeon: case 0x55: *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" + if (Features2 & (1 << (FEATURE_AVX512VNNI - 32))) + *Subtype = INTEL_COREI7_CASCADELAKE; // "cascadelake" + else + *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" break; // Cannonlake: @@ -393,7 +398,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, default: // Unknown family 6 CPU. break; - break; + break; } default: break; // Unknown. @@ -401,8 +406,8 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, - unsigned Features, unsigned *Type, - unsigned *Subtype) { + unsigned Features, unsigned Features2, + unsigned *Type, unsigned *Subtype) { // FIXME: this poorly matches the generated SubtargetFeatureKV table. There // appears to be no way to generate the wide variety of AMD-specific targets // from the information returned from CPUID. @@ -448,7 +453,14 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, break; // "btver2" case 23: *Type = AMDFAM17H; - *Subtype = AMDFAM17H_ZNVER1; + if (Model >= 0x30 && Model <= 0x3f) { + *Subtype = AMDFAM17H_ZNVER2; + break; // "znver2"; 30h-3fh: Zen2 + } + if (Model <= 0x0f) { + *Subtype = AMDFAM17H_ZNVER1; + break; // "znver1"; 00h-0Fh: Zen1 + } break; default: break; // "generic" @@ -462,12 +474,12 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, unsigned Features2 = 0; unsigned EAX, EBX; -#define setFeature(F) \ - do { \ - if (F < 32) \ - Features |= 1U << (F & 0x1f); \ - else if (F < 64) \ - Features2 |= 1U << ((F - 32) & 0x1f); \ +#define setFeature(F) \ + do { \ + if (F < 32) \ + Features |= 1U << (F & 0x1f); \ + else if (F < 64) \ + Features2 |= 1U << ((F - 32) & 0x1f); \ } while (0) if ((EDX >> 15) & 1) @@ -580,24 +592,33 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, #define CONSTRUCTOR_ATTRIBUTE #endif +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif int __cpu_indicator_init(void) CONSTRUCTOR_ATTRIBUTE; +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif struct __processor_model { unsigned int __cpu_vendor; unsigned int __cpu_type; unsigned int __cpu_subtype; unsigned int __cpu_features[1]; } __cpu_model = {0, 0, 0, {0}}; + +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif unsigned int __cpu_features2; -/* A constructor function that is sets __cpu_model and __cpu_features2 with - the right values. This needs to run only once. This constructor is - given the highest priority and it should run before constructors without - the priority set. However, it still runs after ifunc initializers and - needs to be called explicitly there. */ +// A constructor function that is sets __cpu_model and __cpu_features2 with +// the right values. This needs to run only once. This constructor is +// given the highest priority and it should run before constructors without +// the priority set. However, it still runs after ifunc initializers and +// needs to be called explicitly there. -int CONSTRUCTOR_ATTRIBUTE -__cpu_indicator_init(void) { +int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { unsigned EAX, EBX, ECX, EDX; unsigned MaxLeaf = 5; unsigned Vendor; @@ -605,14 +626,14 @@ __cpu_indicator_init(void) { unsigned Features = 0; unsigned Features2 = 0; - /* This function needs to run just once. */ + // This function needs to run just once. if (__cpu_model.__cpu_vendor) return 0; if (!isCpuIdSupported()) return -1; - /* Assume cpuid insn present. Run in level 0 to get vendor id. */ + // Assume cpuid insn present. Run in level 0 to get vendor id. if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) { __cpu_model.__cpu_vendor = VENDOR_OTHER; return -1; @@ -621,20 +642,20 @@ __cpu_indicator_init(void) { detectX86FamilyModel(EAX, &Family, &Model); Brand_id = EBX & 0xff; - /* Find available features. */ + // Find available features. getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2); __cpu_model.__cpu_features[0] = Features; __cpu_features2 = Features2; if (Vendor == SIG_INTEL) { - /* Get CPU type. */ + // Get CPU type. getIntelProcessorTypeAndSubtype(Family, Model, Brand_id, Features, - &(__cpu_model.__cpu_type), + Features2, &(__cpu_model.__cpu_type), &(__cpu_model.__cpu_subtype)); __cpu_model.__cpu_vendor = VENDOR_INTEL; } else if (Vendor == SIG_AMD) { - /* Get CPU type. */ - getAMDProcessorTypeAndSubtype(Family, Model, Features, + // Get CPU type. + getAMDProcessorTypeAndSubtype(Family, Model, Features, Features2, &(__cpu_model.__cpu_type), &(__cpu_model.__cpu_subtype)); __cpu_model.__cpu_vendor = VENDOR_AMD; diff --git a/lib/builtins/ctzdi2.c b/lib/builtins/ctzdi2.c index ef6d7fea1..9384aa605 100644 --- a/lib/builtins/ctzdi2.c +++ b/lib/builtins/ctzdi2.c @@ -1,40 +1,35 @@ -/* ===-- ctzdi2.c - Implement __ctzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzdi2.c - Implement __ctzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits #if !defined(__clang__) && \ - ((defined(__sparc__) && defined(__arch64__)) || \ - defined(__mips64) || \ + ((defined(__sparc__) && defined(__arch64__)) || defined(__mips64) || \ (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) -/* On 64-bit architectures with neither a native clz instruction nor a native - * ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than - * __ctzsi2, leading to infinite recursion. */ +// On 64-bit architectures with neither a native clz instruction nor a native +// ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than +// __ctzsi2, leading to infinite recursion. #define __builtin_ctz(a) __ctzsi2(a) extern si_int __ctzsi2(si_int); #endif -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzdi2(di_int a) -{ - dwords x; - x.all = a; - const si_int f = -(x.s.low == 0); - return __builtin_ctz((x.s.high & f) | (x.s.low & ~f)) + - (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __ctzdi2(di_int a) { + dwords x; + x.all = a; + const si_int f = -(x.s.low == 0); + return __builtin_ctz((x.s.high & f) | (x.s.low & ~f)) + + (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); } diff --git a/lib/builtins/ctzsi2.c b/lib/builtins/ctzsi2.c index c69486ea4..09c6863b7 100644 --- a/lib/builtins/ctzsi2.c +++ b/lib/builtins/ctzsi2.c @@ -1,57 +1,53 @@ -/* ===-- ctzsi2.c - Implement __ctzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzsi2.c - Implement __ctzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzsi2(si_int a) -{ - su_int x = (su_int)a; - si_int t = ((x & 0x0000FFFF) == 0) << 4; /* if (x has no small bits) t = 16 else 0 */ - x >>= t; /* x = [0 - 0xFFFF] + higher garbage bits */ - su_int r = t; /* r = [0, 16] */ - /* return r + ctz(x) */ - t = ((x & 0x00FF) == 0) << 3; - x >>= t; /* x = [0 - 0xFF] + higher garbage bits */ - r += t; /* r = [0, 8, 16, 24] */ - /* return r + ctz(x) */ - t = ((x & 0x0F) == 0) << 2; - x >>= t; /* x = [0 - 0xF] + higher garbage bits */ - r += t; /* r = [0, 4, 8, 12, 16, 20, 24, 28] */ - /* return r + ctz(x) */ - t = ((x & 0x3) == 0) << 1; - x >>= t; - x &= 3; /* x = [0 - 3] */ - r += t; /* r = [0 - 30] and is even */ - /* return r + ctz(x) */ +COMPILER_RT_ABI si_int __ctzsi2(si_int a) { + su_int x = (su_int)a; + si_int t = ((x & 0x0000FFFF) == 0) + << 4; // if (x has no small bits) t = 16 else 0 + x >>= t; // x = [0 - 0xFFFF] + higher garbage bits + su_int r = t; // r = [0, 16] + // return r + ctz(x) + t = ((x & 0x00FF) == 0) << 3; + x >>= t; // x = [0 - 0xFF] + higher garbage bits + r += t; // r = [0, 8, 16, 24] + // return r + ctz(x) + t = ((x & 0x0F) == 0) << 2; + x >>= t; // x = [0 - 0xF] + higher garbage bits + r += t; // r = [0, 4, 8, 12, 16, 20, 24, 28] + // return r + ctz(x) + t = ((x & 0x3) == 0) << 1; + x >>= t; + x &= 3; // x = [0 - 3] + r += t; // r = [0 - 30] and is even + // return r + ctz(x) -/* The branch-less return statement below is equivalent - * to the following switch statement: - * switch (x) - * { - * case 0: - * return r + 2; - * case 2: - * return r + 1; - * case 1: - * case 3: - * return r; - * } - */ - return r + ((2 - (x >> 1)) & -((x & 1) == 0)); + // The branch-less return statement below is equivalent + // to the following switch statement: + // switch (x) + // { + // case 0: + // return r + 2; + // case 2: + // return r + 1; + // case 1: + // case 3: + // return r; + // } + return r + ((2 - (x >> 1)) & -((x & 1) == 0)); } diff --git a/lib/builtins/ctzti2.c b/lib/builtins/ctzti2.c index 45de68270..2a1312c84 100644 --- a/lib/builtins/ctzti2.c +++ b/lib/builtins/ctzti2.c @@ -1,33 +1,29 @@ -/* ===-- ctzti2.c - Implement __ctzti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzti2.c - Implement __ctzti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzti2(ti_int a) -{ - twords x; - x.all = a; - const di_int f = -(x.s.low == 0); - return __builtin_ctzll((x.s.high & f) | (x.s.low & ~f)) + - ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __ctzti2(ti_int a) { + twords x; + x.all = a; + const di_int f = -(x.s.low == 0); + return __builtin_ctzll((x.s.high & f) | (x.s.low & ~f)) + + ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/divdc3.c b/lib/builtins/divdc3.c index 392d6ecac..c2cf62874 100644 --- a/lib/builtins/divdc3.c +++ b/lib/builtins/divdc3.c @@ -1,62 +1,53 @@ -/* ===-- divdc3.c - Implement __divdc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divdc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divdc3.c - Implement __divdc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divdc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Dcomplex -__divdc3(double __a, double __b, double __c, double __d) -{ - int __ilogbw = 0; - double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbn(__c, -__ilogbw); - __d = crt_scalbn(__d, -__ilogbw); +COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c, + double __d) { + int __ilogbw = 0; + double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbn(__c, -__ilogbw); + __d = crt_scalbn(__d, -__ilogbw); + } + double __denom = __c * __c + __d * __d; + Dcomplex z; + COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysign(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysign(crt_isinf(__a) ? 1.0 : 0.0, __a); + __b = crt_copysign(crt_isinf(__b) ? 1.0 : 0.0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0.0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysign(crt_isinf(__c) ? 1.0 : 0.0, __c); + __d = crt_copysign(crt_isinf(__d) ? 1.0 : 0.0, __d); + COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); } - double __denom = __c * __c + __d * __d; - Dcomplex z; - COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysign(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysign(crt_isinf(__a) ? 1.0 : 0.0, __a); - __b = crt_copysign(crt_isinf(__b) ? 1.0 : 0.0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0.0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysign(crt_isinf(__c) ? 1.0 : 0.0, __c); - __d = crt_copysign(crt_isinf(__d) ? 1.0 : 0.0, __d); - COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divdf3.c b/lib/builtins/divdf3.c index 411c82ebb..70c2204a9 100644 --- a/lib/builtins/divdf3.c +++ b/lib/builtins/divdf3.c @@ -1,9 +1,8 @@ //===-- lib/divdf3.c - Double-precision division ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,175 +18,195 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__divdf3(fp_t a, fp_t b) { - - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q31 fixed-point number in the range - // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - const uint32_t q31b = bSignificand >> 21; - uint32_t recip32 = UINT32_C(0x7504f333) - q31b; - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration, so after three iterations, we have about 28 binary - // digits of accuracy. - uint32_t correction32; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - - // recip32 might have overflowed to exactly zero in the preceding - // computation if the high word of b is exactly 1.0. This would sabotage - // the full-width final stage of the computation that follows, so we adjust - // recip32 downward by one bit. - recip32--; - - // We need to perform one more iteration to get us to 56 binary digits; - // The last iteration needs to happen with extra precision. - const uint32_t q63blo = bSignificand << 11; - uint64_t correction, reciprocal; - correction = -((uint64_t)recip32*q31b + ((uint64_t)recip32*q63blo >> 32)); - uint32_t cHi = correction >> 32; - uint32_t cLo = correction; - reciprocal = (uint64_t)recip32*cHi + ((uint64_t)recip32*cLo >> 32); - - // We already adjusted the 32-bit estimate, now we need to adjust the final - // 64-bit reciprocal estimate downward to ensure that it is strictly smaller - // than the infinitely precise exact reciprocal. Because the computation - // of the Newton-Raphson step is truncating at every step, this adjustment - // is small; most of the work is already done. - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-56, lies in the - // interval [0.5, 1.0), and is strictly smaller than the true reciprocal - // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b - // in Q53 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0.5, 2.0) - // 3. the error in q is bounded away from 2^-53 (actually, we have a - // couple of bits to spare, but this is all we need). - - // We need a 64 x 64 multiply high to compute q, which isn't a basic - // operation in C, so we need to be a little bit fussy. - rep_t quotient, quotientLo; - wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - if (quotient < (implicitBit << 1)) { - residual = (aSignificand << 53) - quotient * bSignificand; - quotientExponent--; - } else { - quotient >>= 1; - residual = (aSignificand << 52) - quotient * bSignificand; +COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { + + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - - else { - const bool round = (residual << 1) > bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return - const double result = fromRep(absResult | quotientSign); - return result; + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // one or both of a or b is denormal, the other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Or in the implicit significand bit. (If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything.) + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + + // Align the significand of b as a Q31 fixed-point number in the range + // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + const uint32_t q31b = bSignificand >> 21; + uint32_t recip32 = UINT32_C(0x7504f333) - q31b; + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration, so after three iterations, we have about 28 binary + // digits of accuracy. + uint32_t correction32; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + + // recip32 might have overflowed to exactly zero in the preceding + // computation if the high word of b is exactly 1.0. This would sabotage + // the full-width final stage of the computation that follows, so we adjust + // recip32 downward by one bit. + recip32--; + + // We need to perform one more iteration to get us to 56 binary digits; + // The last iteration needs to happen with extra precision. + const uint32_t q63blo = bSignificand << 11; + uint64_t correction, reciprocal; + correction = -((uint64_t)recip32 * q31b + ((uint64_t)recip32 * q63blo >> 32)); + uint32_t cHi = correction >> 32; + uint32_t cLo = correction; + reciprocal = (uint64_t)recip32 * cHi + ((uint64_t)recip32 * cLo >> 32); + + // We already adjusted the 32-bit estimate, now we need to adjust the final + // 64-bit reciprocal estimate downward to ensure that it is strictly smaller + // than the infinitely precise exact reciprocal. Because the computation + // of the Newton-Raphson step is truncating at every step, this adjustment + // is small; most of the work is already done. + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-56, lies in the + // interval [0.5, 1.0), and is strictly smaller than the true reciprocal + // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b + // in Q53 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0.5, 2.0) + // 3. the error in q is bounded away from 2^-53 (actually, we have a + // couple of bits to spare, but this is all we need). + + // We need a 64 x 64 multiply high to compute q, which isn't a basic + // operation in C, so we need to be a little bit fussy. + rep_t quotient, quotientLo; + wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + if (quotient < (implicitBit << 1)) { + residual = (aSignificand << 53) - quotient * bSignificand; + quotientExponent--; + } else { + quotient >>= 1; + residual = (aSignificand << 52) - quotient * bSignificand; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } + + else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. + return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } + + else { + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit + rep_t absResult = quotient & significandMask; + // Insert the exponent + absResult |= (rep_t)writtenExponent << significandBits; + // Round + absResult += round; + // Insert the sign and return + const double result = fromRep(absResult | quotientSign); + return result; + } } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { - return __divdf3(a, b); -} +AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { return __divdf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divdf3); +COMPILER_RT_ALIAS(__divdf3, __aeabi_ddiv) #endif #endif diff --git a/lib/builtins/divdi3.c b/lib/builtins/divdi3.c index b8eebcb20..ee08d6557 100644 --- a/lib/builtins/divdi3.c +++ b/lib/builtins/divdi3.c @@ -1,29 +1,25 @@ -/* ===-- divdi3.c - Implement __divdi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divdi3.c - Implement __divdi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI di_int -__divdi3(di_int a, di_int b) -{ - const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; - di_int s_a = a >> bits_in_dword_m1; /* s_a = a < 0 ? -1 : 0 */ - di_int s_b = b >> bits_in_dword_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /*sign of quotient */ - return (__udivmoddi4(a, b, (du_int*)0) ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) { + const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; + di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0 + di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + return (__udivmoddi4(a, b, (du_int *)0) ^ s_a) - s_a; // negate if s_a == -1 } diff --git a/lib/builtins/divmoddi4.c b/lib/builtins/divmoddi4.c index 0d4df67a6..7f333510c 100644 --- a/lib/builtins/divmoddi4.c +++ b/lib/builtins/divmoddi4.c @@ -1,25 +1,21 @@ -/*===-- divmoddi4.c - Implement __divmoddi4 --------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divmoddi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divmoddi4.c - Implement __divmoddi4 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI di_int -__divmoddi4(di_int a, di_int b, di_int* rem) -{ - di_int d = __divdi3(a,b); - *rem = a - (d*b); +COMPILER_RT_ABI di_int __divmoddi4(di_int a, di_int b, di_int *rem) { + di_int d = __divdi3(a, b); + *rem = a - (d * b); return d; } diff --git a/lib/builtins/divmodsi4.c b/lib/builtins/divmodsi4.c index dabe28743..402eed22f 100644 --- a/lib/builtins/divmodsi4.c +++ b/lib/builtins/divmodsi4.c @@ -1,27 +1,22 @@ -/*===-- divmodsi4.c - Implement __divmodsi4 --------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divmodsi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divmodsi4.c - Implement __divmodsi4 +//--------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divmodsi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI si_int -__divmodsi4(si_int a, si_int b, si_int* rem) -{ - si_int d = __divsi3(a,b); - *rem = a - (d*b); - return d; +COMPILER_RT_ABI si_int __divmodsi4(si_int a, si_int b, si_int *rem) { + si_int d = __divsi3(a, b); + *rem = a - (d * b); + return d; } - - diff --git a/lib/builtins/divsc3.c b/lib/builtins/divsc3.c index 0d18a256c..1a63634dd 100644 --- a/lib/builtins/divsc3.c +++ b/lib/builtins/divsc3.c @@ -1,63 +1,53 @@ -/*===-- divsc3.c - Implement __divsc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divsc3 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- divsc3.c - Implement __divsc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divsc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Fcomplex -__divsc3(float __a, float __b, float __c, float __d) -{ - int __ilogbw = 0; - float __logbw = - __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnf(__c, -__ilogbw); - __d = crt_scalbnf(__d, -__ilogbw); +COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) { + int __ilogbw = 0; + float __logbw = + __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnf(__c, -__ilogbw); + __d = crt_scalbnf(__d, -__ilogbw); + } + float __denom = __c * __c + __d * __d; + Fcomplex z; + COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignf(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } - float __denom = __c * __c + __d * __d; - Fcomplex z; - COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignf(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); - COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divsf3.c b/lib/builtins/divsf3.c index a74917fd1..9f44fb948 100644 --- a/lib/builtins/divsf3.c +++ b/lib/builtins/divsf3.c @@ -1,9 +1,8 @@ //===-- lib/divsf3.c - Single-precision division ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,159 +18,179 @@ #define SINGLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__divsf3(fp_t a, fp_t b) { - - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q31 fixed-point number in the range - // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - uint32_t q31b = bSignificand << 8; - uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration, so after three iterations, we have about 28 binary - // digits of accuracy. - uint32_t correction; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - - // Exhaustive testing shows that the error in reciprocal after three steps - // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our - // expectations. We bump the reciprocal by a tiny value to force the error - // to be strictly positive (in the range [0x1.4fdfp-37,0x1.287246p-29], to - // be specific). This also causes 1/1 to give a sensible approximation - // instead of zero (due to overflow). - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-28, lies in the - // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller - // than the true reciprocal of b. Multiplying a by this reciprocal thus - // gives a numerical q = a/b in Q24 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0) - // 3. the error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes - // from the fact that we truncate the product, and the 2^27 term - // is the error in the reciprocal of b scaled by the maximum - // possible value of a. As a consequence of this error bound, - // either q or nextafter(q) is the correctly rounded - rep_t quotient = (uint64_t)reciprocal*(aSignificand << 1) >> 32; - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - if (quotient < (implicitBit << 1)) { - residual = (aSignificand << 24) - quotient * bSignificand; - quotientExponent--; - } else { - quotient >>= 1; - residual = (aSignificand << 23) - quotient * bSignificand; +COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { + + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - - else { - const bool round = (residual << 1) > bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // one or both of a or b is denormal, the other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Or in the implicit significand bit. (If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything.) + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + + // Align the significand of b as a Q31 fixed-point number in the range + // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + uint32_t q31b = bSignificand << 8; + uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration, so after three iterations, we have about 28 binary + // digits of accuracy. + uint32_t correction; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + + // Exhaustive testing shows that the error in reciprocal after three steps + // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our + // expectations. We bump the reciprocal by a tiny value to force the error + // to be strictly positive (in the range [0x1.4fdfp-37,0x1.287246p-29], to + // be specific). This also causes 1/1 to give a sensible approximation + // instead of zero (due to overflow). + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-28, lies in the + // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller + // than the true reciprocal of b. Multiplying a by this reciprocal thus + // gives a numerical q = a/b in Q24 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0) + // 3. the error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes + // from the fact that we truncate the product, and the 2^27 term + // is the error in the reciprocal of b scaled by the maximum + // possible value of a. As a consequence of this error bound, + // either q or nextafter(q) is the correctly rounded + rep_t quotient = (uint64_t)reciprocal * (aSignificand << 1) >> 32; + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + if (quotient < (implicitBit << 1)) { + residual = (aSignificand << 24) - quotient * bSignificand; + quotientExponent--; + } else { + quotient >>= 1; + residual = (aSignificand << 23) - quotient * bSignificand; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } + + else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } + + else { + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit + rep_t absResult = quotient & significandMask; + // Insert the exponent + absResult |= (rep_t)writtenExponent << significandBits; + // Round + absResult += round; + // Insert the sign and return + return fromRep(absResult | quotientSign); + } } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { - return __divsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { return __divsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divsf3); +COMPILER_RT_ALIAS(__divsf3, __aeabi_fdiv) #endif #endif diff --git a/lib/builtins/divsi3.c b/lib/builtins/divsi3.c index 75aea008d..b97e11119 100644 --- a/lib/builtins/divsi3.c +++ b/lib/builtins/divsi3.c @@ -1,39 +1,35 @@ -/* ===-- divsi3.c - Implement __divsi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divsi3.c - Implement __divsi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI si_int -__divsi3(si_int a, si_int b) -{ - const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1; - si_int s_a = a >> bits_in_word_m1; /* s_a = a < 0 ? -1 : 0 */ - si_int s_b = b >> bits_in_word_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /* sign of quotient */ - /* - * On CPUs without unsigned hardware division support, - * this calls __udivsi3 (notice the cast to su_int). - * On CPUs with unsigned hardware division support, - * this uses the unsigned division instruction. - */ - return ((su_int)a/(su_int)b ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) { + const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1; + si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0 + si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + // + // On CPUs without unsigned hardware division support, + // this calls __udivsi3 (notice the cast to su_int). + // On CPUs with unsigned hardware division support, + // this uses the unsigned division instruction. + // + return ((su_int)a / (su_int)b ^ s_a) - s_a; // negate if s_a == -1 } #if defined(__ARM_EABI__) -AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) COMPILER_RT_ALIAS(__divsi3); +COMPILER_RT_ALIAS(__divsi3, __aeabi_idiv) #endif diff --git a/lib/builtins/divtc3.c b/lib/builtins/divtc3.c index e5ea00d84..37c71400e 100644 --- a/lib/builtins/divtc3.c +++ b/lib/builtins/divtc3.c @@ -1,63 +1,54 @@ -/*===-- divtc3.c - Implement __divtc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divtc3 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- divtc3.c - Implement __divtc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divtc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Lcomplex -__divtc3(long double __a, long double __b, long double __c, long double __d) -{ - int __ilogbw = 0; - long double __logbw = - __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnl(__c, -__ilogbw); - __d = crt_scalbnl(__d, -__ilogbw); +COMPILER_RT_ABI Lcomplex __divtc3(long double __a, long double __b, + long double __c, long double __d) { + int __ilogbw = 0; + long double __logbw = + __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnl(__c, -__ilogbw); + __d = crt_scalbnl(__d, -__ilogbw); + } + long double __denom = __c * __c + __d * __d; + Lcomplex z; + COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignl(crt_isinf(__a) ? 1.0 : 0.0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1.0 : 0.0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0.0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignl(crt_isinf(__c) ? 1.0 : 0.0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1.0 : 0.0, __d); + COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); } - long double __denom = __c * __c + __d * __d; - Lcomplex z; - COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1.0 : 0.0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1.0 : 0.0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0.0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1.0 : 0.0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1.0 : 0.0, __d); - COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divtf3.c b/lib/builtins/divtf3.c index e81dab826..35e193192 100644 --- a/lib/builtins/divtf3.c +++ b/lib/builtins/divtf3.c @@ -1,9 +1,8 @@ //===-- lib/divtf3.c - Quad-precision division --------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,182 +21,203 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) { - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q63 fixed-point number in the range - // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - const uint64_t q63b = bSignificand >> 49; - uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b; - // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2) - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration. - uint64_t correction64; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - - // recip64 might have overflowed to exactly zero in the preceeding - // computation if the high word of b is exactly 1.0. This would sabotage - // the full-width final stage of the computation that follows, so we adjust - // recip64 downward by one bit. - recip64--; - - // We need to perform one more iteration to get us to 112 binary digits; - // The last iteration needs to happen with extra precision. - const uint64_t q127blo = bSignificand << 15; - rep_t correction, reciprocal; - - // NOTE: This operation is equivalent to __multi3, which is not implemented - // in some architechure - rep_t r64q63, r64q127, r64cH, r64cL, dummy; - wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63); - wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127); - - correction = -(r64q63 + (r64q127 >> 64)); - - uint64_t cHi = correction >> 64; - uint64_t cLo = correction; - - wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH); - wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL); - - reciprocal = r64cH + (r64cL >> 64); - - // We already adjusted the 64-bit estimate, now we need to adjust the final - // 128-bit reciprocal estimate downward to ensure that it is strictly smaller - // than the infinitely precise exact reciprocal. Because the computation - // of the Newton-Raphson step is truncating at every step, this adjustment - // is small; most of the work is already done. - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-112, lies in the - // interval [0.5, 1.0), and is strictly smaller than the true reciprocal - // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b - // in Q127 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0.5, 2.0) - // 3. the error in q is bounded away from 2^-113 (actually, we have a - // couple of bits to spare, but this is all we need). - - // We need a 128 x 128 multiply high to compute q, which isn't a basic - // operation in C, so we need to be a little bit fussy. - rep_t quotient, quotientLo; - wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - rep_t qb; - - if (quotient < (implicitBit << 1)) { - wideMultiply(quotient, bSignificand, &dummy, &qb); - residual = (aSignificand << 113) - qb; - quotientExponent--; - } else { - quotient >>= 1; - wideMultiply(quotient, bSignificand, &dummy, &qb); - residual = (aSignificand << 112) - qb; + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - else { - const bool round = (residual << 1) >= bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return - const long double result = fromRep(absResult | quotientSign); - return result; + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // one or both of a or b is denormal, the other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Or in the implicit significand bit. (If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything.) + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + + // Align the significand of b as a Q63 fixed-point number in the range + // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + const uint64_t q63b = bSignificand >> 49; + uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b; + // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2) + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration. + uint64_t correction64; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + + // recip64 might have overflowed to exactly zero in the preceeding + // computation if the high word of b is exactly 1.0. This would sabotage + // the full-width final stage of the computation that follows, so we adjust + // recip64 downward by one bit. + recip64--; + + // We need to perform one more iteration to get us to 112 binary digits; + // The last iteration needs to happen with extra precision. + const uint64_t q127blo = bSignificand << 15; + rep_t correction, reciprocal; + + // NOTE: This operation is equivalent to __multi3, which is not implemented + // in some architechure + rep_t r64q63, r64q127, r64cH, r64cL, dummy; + wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63); + wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127); + + correction = -(r64q63 + (r64q127 >> 64)); + + uint64_t cHi = correction >> 64; + uint64_t cLo = correction; + + wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH); + wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL); + + reciprocal = r64cH + (r64cL >> 64); + + // We already adjusted the 64-bit estimate, now we need to adjust the final + // 128-bit reciprocal estimate downward to ensure that it is strictly smaller + // than the infinitely precise exact reciprocal. Because the computation + // of the Newton-Raphson step is truncating at every step, this adjustment + // is small; most of the work is already done. + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-112, lies in the + // interval [0.5, 1.0), and is strictly smaller than the true reciprocal + // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b + // in Q127 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0.5, 2.0) + // 3. the error in q is bounded away from 2^-113 (actually, we have a + // couple of bits to spare, but this is all we need). + + // We need a 128 x 128 multiply high to compute q, which isn't a basic + // operation in C, so we need to be a little bit fussy. + rep_t quotient, quotientLo; + wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + rep_t qb; + + if (quotient < (implicitBit << 1)) { + wideMultiply(quotient, bSignificand, &dummy, &qb); + residual = (aSignificand << 113) - qb; + quotientExponent--; + } else { + quotient >>= 1; + wideMultiply(quotient, bSignificand, &dummy, &qb); + residual = (aSignificand << 112) - qb; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. + return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } else { + const bool round = (residual << 1) >= bSignificand; + // Clear the implicit bit + rep_t absResult = quotient & significandMask; + // Insert the exponent + absResult |= (rep_t)writtenExponent << significandBits; + // Round + absResult += round; + // Insert the sign and return + const long double result = fromRep(absResult | quotientSign); + return result; + } } #endif diff --git a/lib/builtins/divti3.c b/lib/builtins/divti3.c index c73eae28f..6d007fe34 100644 --- a/lib/builtins/divti3.c +++ b/lib/builtins/divti3.c @@ -1,33 +1,29 @@ -/* ===-- divti3.c - Implement __divti3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divti3.c - Implement __divti3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI ti_int -__divti3(ti_int a, ti_int b) -{ - const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; - ti_int s_a = a >> bits_in_tword_m1; /* s_a = a < 0 ? -1 : 0 */ - ti_int s_b = b >> bits_in_tword_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /* sign of quotient */ - return (__udivmodti4(a, b, (tu_int*)0) ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) { + const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; + ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0 + ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + return (__udivmodti4(a, b, (tu_int *)0) ^ s_a) - s_a; // negate if s_a == -1 } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/divxc3.c b/lib/builtins/divxc3.c index 6f49280e5..97ffd2eac 100644 --- a/lib/builtins/divxc3.c +++ b/lib/builtins/divxc3.c @@ -1,63 +1,55 @@ -/* ===-- divxc3.c - Implement __divxc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divxc3 for the compiler_rt library. - * - */ +//===-- divxc3.c - Implement __divxc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divxc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Lcomplex -__divxc3(long double __a, long double __b, long double __c, long double __d) -{ - int __ilogbw = 0; - long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnl(__c, -__ilogbw); - __d = crt_scalbnl(__d, -__ilogbw); +COMPILER_RT_ABI Lcomplex __divxc3(long double __a, long double __b, + long double __c, long double __d) { + int __ilogbw = 0; + long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnl(__c, -__ilogbw); + __d = crt_scalbnl(__d, -__ilogbw); + } + long double __denom = __c * __c + __d * __d; + Lcomplex z; + COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } - long double __denom = __c * __c + __d * __d; - Lcomplex z; - COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); - COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } #endif diff --git a/lib/builtins/emutls.c b/lib/builtins/emutls.c index ef95a1c26..da58feb7b 100644 --- a/lib/builtins/emutls.c +++ b/lib/builtins/emutls.c @@ -1,37 +1,35 @@ -/* ===---------- emutls.c - Implements __emutls_get_address ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===---------- emutls.c - Implements __emutls_get_address ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include <stdint.h> #include <stdlib.h> #include <string.h> #include "int_lib.h" -#include "int_util.h" #ifdef __BIONIC__ -/* There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation - to round 2. We need to delay deallocation because: - - Android versions older than M lack __cxa_thread_atexit_impl, so apps - use a pthread key destructor to call C++ destructors. - - Apps might use __thread/thread_local variables in pthread destructors. - We can't wait until the final two rounds, because jemalloc needs two rounds - after the final malloc/free call to free its thread-specific data (see - https://reviews.llvm.org/D46978#1107507). */ +// There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation +// to round 2. We need to delay deallocation because: +// - Android versions older than M lack __cxa_thread_atexit_impl, so apps +// use a pthread key destructor to call C++ destructors. +// - Apps might use __thread/thread_local variables in pthread destructors. +// We can't wait until the final two rounds, because jemalloc needs two rounds +// after the final malloc/free call to free its thread-specific data (see +// https://reviews.llvm.org/D46978#1107507). #define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 1 #else #define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 0 #endif typedef struct emutls_address_array { - uintptr_t skip_destructor_rounds; - uintptr_t size; /* number of elements in the 'data' array */ - void* data[]; + uintptr_t skip_destructor_rounds; + uintptr_t size; // number of elements in the 'data' array + void *data[]; } emutls_address_array; static void emutls_shutdown(emutls_address_array *array); @@ -47,359 +45,339 @@ static bool emutls_key_created = false; typedef unsigned int gcc_word __attribute__((mode(word))); typedef unsigned int gcc_pointer __attribute__((mode(pointer))); -/* Default is not to use posix_memalign, so systems like Android - * can use thread local data without heavier POSIX memory allocators. - */ +// Default is not to use posix_memalign, so systems like Android +// can use thread local data without heavier POSIX memory allocators. #ifndef EMUTLS_USE_POSIX_MEMALIGN #define EMUTLS_USE_POSIX_MEMALIGN 0 #endif static __inline void *emutls_memalign_alloc(size_t align, size_t size) { - void *base; + void *base; #if EMUTLS_USE_POSIX_MEMALIGN - if (posix_memalign(&base, align, size) != 0) - abort(); + if (posix_memalign(&base, align, size) != 0) + abort(); #else - #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) - char* object; - if ((object = (char*)malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) - abort(); - base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) - & ~(uintptr_t)(align - 1)); - - ((void**)base)[-1] = object; +#define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void *)) + char *object; + if ((object = (char *)malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) + abort(); + base = (void *)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) & + ~(uintptr_t)(align - 1)); + + ((void **)base)[-1] = object; #endif - return base; + return base; } static __inline void emutls_memalign_free(void *base) { #if EMUTLS_USE_POSIX_MEMALIGN - free(base); + free(base); #else - /* The mallocated address is in ((void**)base)[-1] */ - free(((void**)base)[-1]); + // The mallocated address is in ((void**)base)[-1] + free(((void **)base)[-1]); #endif } static __inline void emutls_setspecific(emutls_address_array *value) { - pthread_setspecific(emutls_pthread_key, (void*) value); + pthread_setspecific(emutls_pthread_key, (void *)value); } -static __inline emutls_address_array* emutls_getspecific() { - return (emutls_address_array*) pthread_getspecific(emutls_pthread_key); +static __inline emutls_address_array *emutls_getspecific() { + return (emutls_address_array *)pthread_getspecific(emutls_pthread_key); } -static void emutls_key_destructor(void* ptr) { - emutls_address_array *array = (emutls_address_array*)ptr; - if (array->skip_destructor_rounds > 0) { - /* emutls is deallocated using a pthread key destructor. These - * destructors are called in several rounds to accommodate destructor - * functions that (re)initialize key values with pthread_setspecific. - * Delay the emutls deallocation to accommodate other end-of-thread - * cleanup tasks like calling thread_local destructors (e.g. the - * __cxa_thread_atexit fallback in libc++abi). - */ - array->skip_destructor_rounds--; - emutls_setspecific(array); - } else { - emutls_shutdown(array); - free(ptr); - } +static void emutls_key_destructor(void *ptr) { + emutls_address_array *array = (emutls_address_array *)ptr; + if (array->skip_destructor_rounds > 0) { + // emutls is deallocated using a pthread key destructor. These + // destructors are called in several rounds to accommodate destructor + // functions that (re)initialize key values with pthread_setspecific. + // Delay the emutls deallocation to accommodate other end-of-thread + // cleanup tasks like calling thread_local destructors (e.g. the + // __cxa_thread_atexit fallback in libc++abi). + array->skip_destructor_rounds--; + emutls_setspecific(array); + } else { + emutls_shutdown(array); + free(ptr); + } } static __inline void emutls_init(void) { - if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) - abort(); - emutls_key_created = true; + if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) + abort(); + emutls_key_created = true; } static __inline void emutls_init_once(void) { - static pthread_once_t once = PTHREAD_ONCE_INIT; - pthread_once(&once, emutls_init); + static pthread_once_t once = PTHREAD_ONCE_INIT; + pthread_once(&once, emutls_init); } -static __inline void emutls_lock() { - pthread_mutex_lock(&emutls_mutex); -} +static __inline void emutls_lock() { pthread_mutex_lock(&emutls_mutex); } -static __inline void emutls_unlock() { - pthread_mutex_unlock(&emutls_mutex); -} +static __inline void emutls_unlock() { pthread_mutex_unlock(&emutls_mutex); } -#else /* _WIN32 */ +#else // _WIN32 -#include <windows.h> +#include <assert.h> #include <malloc.h> #include <stdio.h> -#include <assert.h> +#include <windows.h> static LPCRITICAL_SECTION emutls_mutex; static DWORD emutls_tls_index = TLS_OUT_OF_INDEXES; typedef uintptr_t gcc_word; -typedef void * gcc_pointer; +typedef void *gcc_pointer; static void win_error(DWORD last_err, const char *hint) { - char *buffer = NULL; - if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_MAX_WIDTH_MASK, - NULL, last_err, 0, (LPSTR)&buffer, 1, NULL)) { - fprintf(stderr, "Windows error: %s\n", buffer); - } else { - fprintf(stderr, "Unkown Windows error: %s\n", hint); - } - LocalFree(buffer); + char *buffer = NULL; + if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_MAX_WIDTH_MASK, + NULL, last_err, 0, (LPSTR)&buffer, 1, NULL)) { + fprintf(stderr, "Windows error: %s\n", buffer); + } else { + fprintf(stderr, "Unkown Windows error: %s\n", hint); + } + LocalFree(buffer); } static __inline void win_abort(DWORD last_err, const char *hint) { - win_error(last_err, hint); - abort(); + win_error(last_err, hint); + abort(); } static __inline void *emutls_memalign_alloc(size_t align, size_t size) { - void *base = _aligned_malloc(size, align); - if (!base) - win_abort(GetLastError(), "_aligned_malloc"); - return base; + void *base = _aligned_malloc(size, align); + if (!base) + win_abort(GetLastError(), "_aligned_malloc"); + return base; } -static __inline void emutls_memalign_free(void *base) { - _aligned_free(base); -} +static __inline void emutls_memalign_free(void *base) { _aligned_free(base); } static void emutls_exit(void) { - if (emutls_mutex) { - DeleteCriticalSection(emutls_mutex); - _aligned_free(emutls_mutex); - emutls_mutex = NULL; - } - if (emutls_tls_index != TLS_OUT_OF_INDEXES) { - emutls_shutdown((emutls_address_array*)TlsGetValue(emutls_tls_index)); - TlsFree(emutls_tls_index); - emutls_tls_index = TLS_OUT_OF_INDEXES; - } + if (emutls_mutex) { + DeleteCriticalSection(emutls_mutex); + _aligned_free(emutls_mutex); + emutls_mutex = NULL; + } + if (emutls_tls_index != TLS_OUT_OF_INDEXES) { + emutls_shutdown((emutls_address_array *)TlsGetValue(emutls_tls_index)); + TlsFree(emutls_tls_index); + emutls_tls_index = TLS_OUT_OF_INDEXES; + } } -#pragma warning (push) -#pragma warning (disable : 4100) +#pragma warning(push) +#pragma warning(disable : 4100) static BOOL CALLBACK emutls_init(PINIT_ONCE p0, PVOID p1, PVOID *p2) { - emutls_mutex = (LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16); - if (!emutls_mutex) { - win_error(GetLastError(), "_aligned_malloc"); - return FALSE; - } - InitializeCriticalSection(emutls_mutex); - - emutls_tls_index = TlsAlloc(); - if (emutls_tls_index == TLS_OUT_OF_INDEXES) { - emutls_exit(); - win_error(GetLastError(), "TlsAlloc"); - return FALSE; - } - atexit(&emutls_exit); - return TRUE; + emutls_mutex = + (LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16); + if (!emutls_mutex) { + win_error(GetLastError(), "_aligned_malloc"); + return FALSE; + } + InitializeCriticalSection(emutls_mutex); + + emutls_tls_index = TlsAlloc(); + if (emutls_tls_index == TLS_OUT_OF_INDEXES) { + emutls_exit(); + win_error(GetLastError(), "TlsAlloc"); + return FALSE; + } + atexit(&emutls_exit); + return TRUE; } static __inline void emutls_init_once(void) { - static INIT_ONCE once; - InitOnceExecuteOnce(&once, emutls_init, NULL, NULL); + static INIT_ONCE once; + InitOnceExecuteOnce(&once, emutls_init, NULL, NULL); } -static __inline void emutls_lock() { - EnterCriticalSection(emutls_mutex); -} +static __inline void emutls_lock() { EnterCriticalSection(emutls_mutex); } -static __inline void emutls_unlock() { - LeaveCriticalSection(emutls_mutex); -} +static __inline void emutls_unlock() { LeaveCriticalSection(emutls_mutex); } static __inline void emutls_setspecific(emutls_address_array *value) { - if (TlsSetValue(emutls_tls_index, (LPVOID) value) == 0) - win_abort(GetLastError(), "TlsSetValue"); + if (TlsSetValue(emutls_tls_index, (LPVOID)value) == 0) + win_abort(GetLastError(), "TlsSetValue"); } -static __inline emutls_address_array* emutls_getspecific() { - LPVOID value = TlsGetValue(emutls_tls_index); - if (value == NULL) { - const DWORD err = GetLastError(); - if (err != ERROR_SUCCESS) - win_abort(err, "TlsGetValue"); - } - return (emutls_address_array*) value; +static __inline emutls_address_array *emutls_getspecific() { + LPVOID value = TlsGetValue(emutls_tls_index); + if (value == NULL) { + const DWORD err = GetLastError(); + if (err != ERROR_SUCCESS) + win_abort(err, "TlsGetValue"); + } + return (emutls_address_array *)value; } -/* Provide atomic load/store functions for emutls_get_index if built with MSVC. - */ +// Provide atomic load/store functions for emutls_get_index if built with MSVC. #if !defined(__ATOMIC_RELEASE) #include <intrin.h> enum { __ATOMIC_ACQUIRE = 2, __ATOMIC_RELEASE = 3 }; static __inline uintptr_t __atomic_load_n(void *ptr, unsigned type) { - assert(type == __ATOMIC_ACQUIRE); - // These return the previous value - but since we do an OR with 0, - // it's equivalent to a plain load. + assert(type == __ATOMIC_ACQUIRE); + // These return the previous value - but since we do an OR with 0, + // it's equivalent to a plain load. #ifdef _WIN64 - return InterlockedOr64(ptr, 0); + return InterlockedOr64(ptr, 0); #else - return InterlockedOr(ptr, 0); + return InterlockedOr(ptr, 0); #endif } static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) { - assert(type == __ATOMIC_RELEASE); - InterlockedExchangePointer((void *volatile *)ptr, (void *)val); + assert(type == __ATOMIC_RELEASE); + InterlockedExchangePointer((void *volatile *)ptr, (void *)val); } -#endif /* __ATOMIC_RELEASE */ +#endif // __ATOMIC_RELEASE -#pragma warning (pop) +#pragma warning(pop) -#endif /* _WIN32 */ +#endif // _WIN32 -static size_t emutls_num_object = 0; /* number of allocated TLS objects */ +static size_t emutls_num_object = 0; // number of allocated TLS objects -/* Free the allocated TLS data - */ +// Free the allocated TLS data static void emutls_shutdown(emutls_address_array *array) { - if (array) { - uintptr_t i; - for (i = 0; i < array->size; ++i) { - if (array->data[i]) - emutls_memalign_free(array->data[i]); - } + if (array) { + uintptr_t i; + for (i = 0; i < array->size; ++i) { + if (array->data[i]) + emutls_memalign_free(array->data[i]); } + } } -/* For every TLS variable xyz, - * there is one __emutls_control variable named __emutls_v.xyz. - * If xyz has non-zero initial value, __emutls_v.xyz's "value" - * will point to __emutls_t.xyz, which has the initial value. - */ +// For every TLS variable xyz, +// there is one __emutls_control variable named __emutls_v.xyz. +// If xyz has non-zero initial value, __emutls_v.xyz's "value" +// will point to __emutls_t.xyz, which has the initial value. typedef struct __emutls_control { - /* Must use gcc_word here, instead of size_t, to match GCC. When - gcc_word is larger than size_t, the upper extra bits are all - zeros. We can use variables of size_t to operate on size and - align. */ - gcc_word size; /* size of the object in bytes */ - gcc_word align; /* alignment of the object in bytes */ - union { - uintptr_t index; /* data[index-1] is the object address */ - void* address; /* object address, when in single thread env */ - } object; - void* value; /* null or non-zero initial value for the object */ + // Must use gcc_word here, instead of size_t, to match GCC. When + // gcc_word is larger than size_t, the upper extra bits are all + // zeros. We can use variables of size_t to operate on size and + // align. + gcc_word size; // size of the object in bytes + gcc_word align; // alignment of the object in bytes + union { + uintptr_t index; // data[index-1] is the object address + void *address; // object address, when in single thread env + } object; + void *value; // null or non-zero initial value for the object } __emutls_control; -/* Emulated TLS objects are always allocated at run-time. */ +// Emulated TLS objects are always allocated at run-time. static __inline void *emutls_allocate_object(__emutls_control *control) { - /* Use standard C types, check with gcc's emutls.o. */ - COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); - COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); - - size_t size = control->size; - size_t align = control->align; - void* base; - if (align < sizeof(void*)) - align = sizeof(void*); - /* Make sure that align is power of 2. */ - if ((align & (align - 1)) != 0) - abort(); - - base = emutls_memalign_alloc(align, size); - if (control->value) - memcpy(base, control->value, size); - else - memset(base, 0, size); - return base; -} + // Use standard C types, check with gcc's emutls.o. + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void *)); + + size_t size = control->size; + size_t align = control->align; + void *base; + if (align < sizeof(void *)) + align = sizeof(void *); + // Make sure that align is power of 2. + if ((align & (align - 1)) != 0) + abort(); + base = emutls_memalign_alloc(align, size); + if (control->value) + memcpy(base, control->value, size); + else + memset(base, 0, size); + return base; +} -/* Returns control->object.index; set index if not allocated yet. */ +// Returns control->object.index; set index if not allocated yet. static __inline uintptr_t emutls_get_index(__emutls_control *control) { - uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); + uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); + if (!index) { + emutls_init_once(); + emutls_lock(); + index = control->object.index; if (!index) { - emutls_init_once(); - emutls_lock(); - index = control->object.index; - if (!index) { - index = ++emutls_num_object; - __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); - } - emutls_unlock(); + index = ++emutls_num_object; + __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); } - return index; + emutls_unlock(); + } + return index; } -/* Updates newly allocated thread local emutls_address_array. */ +// Updates newly allocated thread local emutls_address_array. static __inline void emutls_check_array_set_size(emutls_address_array *array, uintptr_t size) { - if (array == NULL) - abort(); - array->size = size; - emutls_setspecific(array); + if (array == NULL) + abort(); + array->size = size; + emutls_setspecific(array); } -/* Returns the new 'data' array size, number of elements, - * which must be no smaller than the given index. - */ +// Returns the new 'data' array size, number of elements, +// which must be no smaller than the given index. static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { - /* Need to allocate emutls_address_array with extra slots - * to store the header. - * Round up the emutls_address_array size to multiple of 16. - */ - uintptr_t header_words = sizeof(emutls_address_array) / sizeof(void *); - return ((index + header_words + 15) & ~((uintptr_t)15)) - header_words; + // Need to allocate emutls_address_array with extra slots + // to store the header. + // Round up the emutls_address_array size to multiple of 16. + uintptr_t header_words = sizeof(emutls_address_array) / sizeof(void *); + return ((index + header_words + 15) & ~((uintptr_t)15)) - header_words; } -/* Returns the size in bytes required for an emutls_address_array with - * N number of elements for data field. - */ +// Returns the size in bytes required for an emutls_address_array with +// N number of elements for data field. static __inline uintptr_t emutls_asize(uintptr_t N) { - return N * sizeof(void *) + sizeof(emutls_address_array); + return N * sizeof(void *) + sizeof(emutls_address_array); } -/* Returns the thread local emutls_address_array. - * Extends its size if necessary to hold address at index. - */ +// Returns the thread local emutls_address_array. +// Extends its size if necessary to hold address at index. static __inline emutls_address_array * emutls_get_address_array(uintptr_t index) { - emutls_address_array* array = emutls_getspecific(); - if (array == NULL) { - uintptr_t new_size = emutls_new_data_array_size(index); - array = (emutls_address_array*) malloc(emutls_asize(new_size)); - if (array) { - memset(array->data, 0, new_size * sizeof(void*)); - array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS; - } - emutls_check_array_set_size(array, new_size); - } else if (index > array->size) { - uintptr_t orig_size = array->size; - uintptr_t new_size = emutls_new_data_array_size(index); - array = (emutls_address_array*) realloc(array, emutls_asize(new_size)); - if (array) - memset(array->data + orig_size, 0, - (new_size - orig_size) * sizeof(void*)); - emutls_check_array_set_size(array, new_size); + emutls_address_array *array = emutls_getspecific(); + if (array == NULL) { + uintptr_t new_size = emutls_new_data_array_size(index); + array = (emutls_address_array *)malloc(emutls_asize(new_size)); + if (array) { + memset(array->data, 0, new_size * sizeof(void *)); + array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS; } - return array; + emutls_check_array_set_size(array, new_size); + } else if (index > array->size) { + uintptr_t orig_size = array->size; + uintptr_t new_size = emutls_new_data_array_size(index); + array = (emutls_address_array *)realloc(array, emutls_asize(new_size)); + if (array) + memset(array->data + orig_size, 0, + (new_size - orig_size) * sizeof(void *)); + emutls_check_array_set_size(array, new_size); + } + return array; } -void* __emutls_get_address(__emutls_control* control) { - uintptr_t index = emutls_get_index(control); - emutls_address_array* array = emutls_get_address_array(index--); - if (array->data[index] == NULL) - array->data[index] = emutls_allocate_object(control); - return array->data[index]; +void *__emutls_get_address(__emutls_control *control) { + uintptr_t index = emutls_get_index(control); + emutls_address_array *array = emutls_get_address_array(index--); + if (array->data[index] == NULL) + array->data[index] = emutls_allocate_object(control); + return array->data[index]; } #ifdef __BIONIC__ -/* Called by Bionic on dlclose to delete the emutls pthread key. */ -__attribute__((visibility("hidden"))) -void __emutls_unregister_key(void) { - if (emutls_key_created) { - pthread_key_delete(emutls_pthread_key); - emutls_key_created = false; - } +// Called by Bionic on dlclose to delete the emutls pthread key. +__attribute__((visibility("hidden"))) void __emutls_unregister_key(void) { + if (emutls_key_created) { + pthread_key_delete(emutls_pthread_key); + emutls_key_created = false; + } } #endif diff --git a/lib/builtins/enable_execute_stack.c b/lib/builtins/enable_execute_stack.c index 327d460b4..e18de4eae 100644 --- a/lib/builtins/enable_execute_stack.c +++ b/lib/builtins/enable_execute_stack.c @@ -1,12 +1,10 @@ -/* ===-- enable_execute_stack.c - Implement __enable_execute_stack ---------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- enable_execute_stack.c - Implement __enable_execute_stack ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -14,10 +12,9 @@ #include <sys/mman.h> #endif -/* #include "config.h" - * FIXME: CMake - include when cmake system is ready. - * Remove #define HAVE_SYSCONF 1 line. - */ +// #include "config.h" +// FIXME: CMake - include when cmake system is ready. +// Remove #define HAVE_SYSCONF 1 line. #define HAVE_SYSCONF 1 #ifdef _WIN32 @@ -26,47 +23,45 @@ #else #ifndef __APPLE__ #include <unistd.h> -#endif /* __APPLE__ */ -#endif /* _WIN32 */ +#endif // __APPLE__ +#endif // _WIN32 #if __LP64__ - #define TRAMPOLINE_SIZE 48 +#define TRAMPOLINE_SIZE 48 #else - #define TRAMPOLINE_SIZE 40 +#define TRAMPOLINE_SIZE 40 #endif -/* - * The compiler generates calls to __enable_execute_stack() when creating - * trampoline functions on the stack for use with nested functions. - * It is expected to mark the page(s) containing the address - * and the next 48 bytes as executable. Since the stack is normally rw- - * that means changing the protection on those page(s) to rwx. - */ +// The compiler generates calls to __enable_execute_stack() when creating +// trampoline functions on the stack for use with nested functions. +// It is expected to mark the page(s) containing the address +// and the next 48 bytes as executable. Since the stack is normally rw- +// that means changing the protection on those page(s) to rwx. -COMPILER_RT_ABI void -__enable_execute_stack(void* addr) -{ +COMPILER_RT_ABI void __enable_execute_stack(void *addr) { #if _WIN32 - MEMORY_BASIC_INFORMATION mbi; - if (!VirtualQuery (addr, &mbi, sizeof(mbi))) - return; /* We should probably assert here because there is no return value */ - VirtualProtect (mbi.BaseAddress, mbi.RegionSize, PAGE_EXECUTE_READWRITE, &mbi.Protect); + MEMORY_BASIC_INFORMATION mbi; + if (!VirtualQuery(addr, &mbi, sizeof(mbi))) + return; // We should probably assert here because there is no return value + VirtualProtect(mbi.BaseAddress, mbi.RegionSize, PAGE_EXECUTE_READWRITE, + &mbi.Protect); #else #if __APPLE__ - /* On Darwin, pagesize is always 4096 bytes */ - const uintptr_t pageSize = 4096; + // On Darwin, pagesize is always 4096 bytes + const uintptr_t pageSize = 4096; #elif !defined(HAVE_SYSCONF) #error "HAVE_SYSCONF not defined! See enable_execute_stack.c" #else - const uintptr_t pageSize = sysconf(_SC_PAGESIZE); -#endif /* __APPLE__ */ + const uintptr_t pageSize = sysconf(_SC_PAGESIZE); +#endif // __APPLE__ - const uintptr_t pageAlignMask = ~(pageSize-1); - uintptr_t p = (uintptr_t)addr; - unsigned char* startPage = (unsigned char*)(p & pageAlignMask); - unsigned char* endPage = (unsigned char*)((p+TRAMPOLINE_SIZE+pageSize) & pageAlignMask); - size_t length = endPage - startPage; - (void) mprotect((void *)startPage, length, PROT_READ | PROT_WRITE | PROT_EXEC); + const uintptr_t pageAlignMask = ~(pageSize - 1); + uintptr_t p = (uintptr_t)addr; + unsigned char *startPage = (unsigned char *)(p & pageAlignMask); + unsigned char *endPage = + (unsigned char *)((p + TRAMPOLINE_SIZE + pageSize) & pageAlignMask); + size_t length = endPage - startPage; + (void)mprotect((void *)startPage, length, PROT_READ | PROT_WRITE | PROT_EXEC); #endif } diff --git a/lib/builtins/eprintf.c b/lib/builtins/eprintf.c index 89f34b154..89fb0e315 100644 --- a/lib/builtins/eprintf.c +++ b/lib/builtins/eprintf.c @@ -1,35 +1,27 @@ -/* ===---------- eprintf.c - Implements __eprintf --------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ - - +//===---------- eprintf.c - Implements __eprintf --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <stdio.h> - -/* - * __eprintf() was used in an old version of <assert.h>. - * It can eventually go away, but it is needed when linking - * .o files built with the old <assert.h>. - * - * It should never be exported from a dylib, so it is marked - * visibility hidden. - */ +// __eprintf() was used in an old version of <assert.h>. +// It can eventually go away, but it is needed when linking +// .o files built with the old <assert.h>. +// +// It should never be exported from a dylib, so it is marked +// visibility hidden. #ifndef _WIN32 __attribute__((visibility("hidden"))) #endif COMPILER_RT_ABI void -__eprintf(const char* format, const char* assertion_expression, - const char* line, const char* file) -{ - fprintf(stderr, format, assertion_expression, line, file); - fflush(stderr); - compilerrt_abort(); +__eprintf(const char *format, const char *assertion_expression, + const char *line, const char *file) { + fprintf(stderr, format, assertion_expression, line, file); + fflush(stderr); + compilerrt_abort(); } diff --git a/lib/builtins/extenddftf2.c b/lib/builtins/extenddftf2.c index 86dab8f03..849a39da1 100644 --- a/lib/builtins/extenddftf2.c +++ b/lib/builtins/extenddftf2.c @@ -1,12 +1,10 @@ //===-- lib/extenddftf2.c - double -> quad conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define QUAD_PRECISION #include "fp_lib.h" @@ -17,7 +15,7 @@ #include "fp_extend_impl.inc" COMPILER_RT_ABI long double __extenddftf2(double a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } #endif diff --git a/lib/builtins/extendhfsf2.c b/lib/builtins/extendhfsf2.c index d9c0db84b..7c1a76eb5 100644 --- a/lib/builtins/extendhfsf2.c +++ b/lib/builtins/extendhfsf2.c @@ -1,12 +1,10 @@ //===-- lib/extendhfsf2.c - half -> single conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define SRC_HALF #define DST_SINGLE @@ -15,19 +13,15 @@ // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } -COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { - return __extendhfsf2(a); -} +COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { return __extendhfsf2(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_h2f(uint16_t a) { - return __extendhfsf2(a); -} +AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); } #else -AEABI_RTABI float __aeabi_h2f(uint16_t a) COMPILER_RT_ALIAS(__extendhfsf2); +COMPILER_RT_ALIAS(__extendhfsf2, __aeabi_h2f) #endif #endif diff --git a/lib/builtins/extendsfdf2.c b/lib/builtins/extendsfdf2.c index 3d84529a6..8132d57e6 100644 --- a/lib/builtins/extendsfdf2.c +++ b/lib/builtins/extendsfdf2.c @@ -1,27 +1,21 @@ //===-- lib/extendsfdf2.c - single -> double conversion -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define SRC_SINGLE #define DST_DOUBLE #include "fp_extend_impl.inc" -COMPILER_RT_ABI double __extendsfdf2(float a) { - return __extendXfYf2__(a); -} +COMPILER_RT_ABI double __extendsfdf2(float a) { return __extendXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_f2d(float a) { - return __extendsfdf2(a); -} +AEABI_RTABI double __aeabi_f2d(float a) { return __extendsfdf2(a); } #else -AEABI_RTABI double __aeabi_f2d(float a) COMPILER_RT_ALIAS(__extendsfdf2); +COMPILER_RT_ALIAS(__extendsfdf2, __aeabi_f2d) #endif #endif diff --git a/lib/builtins/extendsftf2.c b/lib/builtins/extendsftf2.c index 2eeeba284..c6368406d 100644 --- a/lib/builtins/extendsftf2.c +++ b/lib/builtins/extendsftf2.c @@ -1,12 +1,10 @@ //===-- lib/extendsftf2.c - single -> quad conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define QUAD_PRECISION #include "fp_lib.h" @@ -17,7 +15,7 @@ #include "fp_extend_impl.inc" COMPILER_RT_ABI long double __extendsftf2(float a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } #endif diff --git a/lib/builtins/ffsdi2.c b/lib/builtins/ffsdi2.c index a5ac9900f..9c1a24260 100644 --- a/lib/builtins/ffsdi2.c +++ b/lib/builtins/ffsdi2.c @@ -1,33 +1,27 @@ -/* ===-- ffsdi2.c - Implement __ffsdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffsdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffsdi2.c - Implement __ffsdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffsdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffsdi2(di_int a) -{ - dwords x; - x.all = a; - if (x.s.low == 0) - { - if (x.s.high == 0) - return 0; - return __builtin_ctz(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT); - } - return __builtin_ctz(x.s.low) + 1; +COMPILER_RT_ABI si_int __ffsdi2(di_int a) { + dwords x; + x.all = a; + if (x.s.low == 0) { + if (x.s.high == 0) + return 0; + return __builtin_ctz(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT); + } + return __builtin_ctz(x.s.low) + 1; } diff --git a/lib/builtins/ffssi2.c b/lib/builtins/ffssi2.c index e5180eff5..cba1f72fd 100644 --- a/lib/builtins/ffssi2.c +++ b/lib/builtins/ffssi2.c @@ -1,29 +1,23 @@ -/* ===-- ffssi2.c - Implement __ffssi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffssi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffssi2.c - Implement __ffssi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffssi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffssi2(si_int a) -{ - if (a == 0) - { - return 0; - } - return __builtin_ctz(a) + 1; +COMPILER_RT_ABI si_int __ffssi2(si_int a) { + if (a == 0) { + return 0; + } + return __builtin_ctz(a) + 1; } diff --git a/lib/builtins/ffsti2.c b/lib/builtins/ffsti2.c index dcdb3bd7f..a2d7ce08a 100644 --- a/lib/builtins/ffsti2.c +++ b/lib/builtins/ffsti2.c @@ -1,37 +1,31 @@ -/* ===-- ffsti2.c - Implement __ffsti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffsti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffsti2.c - Implement __ffsti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffsti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffsti2(ti_int a) -{ - twords x; - x.all = a; - if (x.s.low == 0) - { - if (x.s.high == 0) - return 0; - return __builtin_ctzll(x.s.high) + (1 + sizeof(di_int) * CHAR_BIT); - } - return __builtin_ctzll(x.s.low) + 1; +COMPILER_RT_ABI si_int __ffsti2(ti_int a) { + twords x; + x.all = a; + if (x.s.low == 0) { + if (x.s.high == 0) + return 0; + return __builtin_ctzll(x.s.high) + (1 + sizeof(di_int) * CHAR_BIT); + } + return __builtin_ctzll(x.s.low) + 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixdfdi.c b/lib/builtins/fixdfdi.c index 54e312d3c..2ed5261c5 100644 --- a/lib/builtins/fixdfdi.c +++ b/lib/builtins/fixdfdi.c @@ -1,55 +1,44 @@ -/* ===-- fixdfdi.c - Implement __fixdfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfdi.c - Implement __fixdfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. COMPILER_RT_ABI du_int __fixunsdfdi(double a); -COMPILER_RT_ABI di_int -__fixdfdi(double a) -{ - if (a < 0.0) { - return -__fixunsdfdi(-a); - } - return __fixunsdfdi(a); +COMPILER_RT_ABI di_int __fixdfdi(double a) { + if (a < 0.0) { + return -__fixunsdfdi(-a); + } + return __fixunsdfdi(a); } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixdfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixdfdi(fp_t a) { return __fixint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { - return __fixdfdi(a); -} +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { return __fixdfdi(a); } #else -AEABI_RTABI di_int __aeabi_d2lz(fp_t a) COMPILER_RT_ALIAS(__fixdfdi); +COMPILER_RT_ALIAS(__fixdfdi, __aeabi_d2lz) #endif #endif diff --git a/lib/builtins/fixdfsi.c b/lib/builtins/fixdfsi.c index 5b9588175..f54649993 100644 --- a/lib/builtins/fixdfsi.c +++ b/lib/builtins/fixdfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixdfsi.c - Implement __fixdfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfsi.c - Implement __fixdfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" @@ -14,17 +12,12 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixdfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixdfsi(fp_t a) { return __fixint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { - return __fixdfsi(a); -} +AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { return __fixdfsi(a); } #else -AEABI_RTABI si_int __aeabi_d2iz(fp_t a) COMPILER_RT_ALIAS(__fixdfsi); +COMPILER_RT_ALIAS(__fixdfsi, __aeabi_d2iz) #endif #endif diff --git a/lib/builtins/fixdfti.c b/lib/builtins/fixdfti.c index aaf225e74..90ca8959d 100644 --- a/lib/builtins/fixdfti.c +++ b/lib/builtins/fixdfti.c @@ -1,12 +1,10 @@ -/* ===-- fixdfti.c - Implement __fixdfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfti.c - Implement __fixdfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -18,9 +16,6 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixdfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixdfti(fp_t a) { return __fixint(a); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixsfdi.c b/lib/builtins/fixsfdi.c index 32e87c608..615e93d4f 100644 --- a/lib/builtins/fixsfdi.c +++ b/lib/builtins/fixsfdi.c @@ -1,55 +1,44 @@ -/* ===-- fixsfdi.c - Implement __fixsfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfdi.c - Implement __fixsfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. COMPILER_RT_ABI du_int __fixunssfdi(float a); -COMPILER_RT_ABI di_int -__fixsfdi(float a) -{ - if (a < 0.0f) { - return -__fixunssfdi(-a); - } - return __fixunssfdi(a); +COMPILER_RT_ABI di_int __fixsfdi(float a) { + if (a < 0.0f) { + return -__fixunssfdi(-a); + } + return __fixunssfdi(a); } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixsfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixsfdi(fp_t a) { return __fixint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { - return __fixsfdi(a); -} +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { return __fixsfdi(a); } #else -AEABI_RTABI di_int __aeabi_f2lz(fp_t a) COMPILER_RT_ALIAS(__fixsfdi); +COMPILER_RT_ALIAS(__fixsfdi, __aeabi_f2lz) #endif #endif diff --git a/lib/builtins/fixsfsi.c b/lib/builtins/fixsfsi.c index e94e5f3dc..d83d7e722 100644 --- a/lib/builtins/fixsfsi.c +++ b/lib/builtins/fixsfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixsfsi.c - Implement __fixsfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfsi.c - Implement __fixsfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" @@ -14,17 +12,12 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixsfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixsfsi(fp_t a) { return __fixint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { - return __fixsfsi(a); -} +AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { return __fixsfsi(a); } #else -AEABI_RTABI si_int __aeabi_f2iz(fp_t a) COMPILER_RT_ALIAS(__fixsfsi); +COMPILER_RT_ALIAS(__fixsfsi, __aeabi_f2iz) #endif #endif diff --git a/lib/builtins/fixsfti.c b/lib/builtins/fixsfti.c index 3a159b3e1..3c01b75e2 100644 --- a/lib/builtins/fixsfti.c +++ b/lib/builtins/fixsfti.c @@ -1,12 +1,10 @@ -/* ===-- fixsfti.c - Implement __fixsfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfti.c - Implement __fixsfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -18,9 +16,6 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixsfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixsfti(fp_t a) { return __fixint(a); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixtfdi.c b/lib/builtins/fixtfdi.c index bc9dea1f4..fe570e6b3 100644 --- a/lib/builtins/fixtfdi.c +++ b/lib/builtins/fixtfdi.c @@ -1,12 +1,10 @@ -/* ===-- fixtfdi.c - Implement __fixtfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfdi.c - Implement __fixtfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixtfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixtfdi(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixtfsi.c b/lib/builtins/fixtfsi.c index feb3de885..a32bd964c 100644 --- a/lib/builtins/fixtfsi.c +++ b/lib/builtins/fixtfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixtfsi.c - Implement __fixtfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfsi.c - Implement __fixtfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixtfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixtfsi(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixtfti.c b/lib/builtins/fixtfti.c index ee4ada85c..19f84ce38 100644 --- a/lib/builtins/fixtfti.c +++ b/lib/builtins/fixtfti.c @@ -1,12 +1,10 @@ -/* ===-- fixtfti.c - Implement __fixtfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfti.c - Implement __fixtfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixtfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixtfti(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixunsdfdi.c b/lib/builtins/fixunsdfdi.c index bfe4dbb25..d2ba73825 100644 --- a/lib/builtins/fixunsdfdi.c +++ b/lib/builtins/fixunsdfdi.c @@ -1,52 +1,42 @@ -/* ===-- fixunsdfdi.c - Implement __fixunsdfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfdi.c - Implement __fixunsdfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ - -COMPILER_RT_ABI du_int -__fixunsdfdi(double a) -{ - if (a <= 0.0) return 0; - su_int high = a / 4294967296.f; /* a / 0x1p32f; */ - su_int low = a - (double)high * 4294967296.f; /* high * 0x1p32f; */ - return ((du_int)high << 32) | low; +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. + +COMPILER_RT_ABI du_int __fixunsdfdi(double a) { + if (a <= 0.0) + return 0; + su_int high = a / 4294967296.f; // a / 0x1p32f; + su_int low = a - (double)high * 4294967296.f; // high * 0x1p32f; + return ((du_int)high << 32) | low; } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunsdfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunsdfdi(fp_t a) { return __fixuint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { - return __fixunsdfdi(a); -} +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { return __fixunsdfdi(a); } #else -AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfdi); +COMPILER_RT_ALIAS(__fixunsdfdi, __aeabi_d2ulz) #endif #endif diff --git a/lib/builtins/fixunsdfsi.c b/lib/builtins/fixunsdfsi.c index 3c5355bea..3db2adec0 100644 --- a/lib/builtins/fixunsdfsi.c +++ b/lib/builtins/fixunsdfsi.c @@ -1,29 +1,22 @@ -/* ===-- fixunsdfsi.c - Implement __fixunsdfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfsi.c - Implement __fixunsdfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunsdfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunsdfsi(fp_t a) { return __fixuint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { - return __fixunsdfsi(a); -} +AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { return __fixunsdfsi(a); } #else -AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfsi); +COMPILER_RT_ALIAS(__fixunsdfsi, __aeabi_d2uiz) #endif #endif diff --git a/lib/builtins/fixunsdfti.c b/lib/builtins/fixunsdfti.c index f8046a026..be497d0e1 100644 --- a/lib/builtins/fixunsdfti.c +++ b/lib/builtins/fixunsdfti.c @@ -1,12 +1,10 @@ -/* ===-- fixunsdfti.c - Implement __fixunsdfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfti.c - Implement __fixunsdfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -16,8 +14,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunsdfti(fp_t a) { - return __fixuint(a); -} -#endif /* CRT_HAS_128BIT */ +COMPILER_RT_ABI tu_int __fixunsdfti(fp_t a) { return __fixuint(a); } +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixunssfdi.c b/lib/builtins/fixunssfdi.c index 080a25bb1..2b90dafad 100644 --- a/lib/builtins/fixunssfdi.c +++ b/lib/builtins/fixunssfdi.c @@ -1,53 +1,43 @@ -/* ===-- fixunssfdi.c - Implement __fixunssfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfdi.c - Implement __fixunssfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ - -COMPILER_RT_ABI du_int -__fixunssfdi(float a) -{ - if (a <= 0.0f) return 0; - double da = a; - su_int high = da / 4294967296.f; /* da / 0x1p32f; */ - su_int low = da - (double)high * 4294967296.f; /* high * 0x1p32f; */ - return ((du_int)high << 32) | low; +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. + +COMPILER_RT_ABI du_int __fixunssfdi(float a) { + if (a <= 0.0f) + return 0; + double da = a; + su_int high = da / 4294967296.f; // da / 0x1p32f; + su_int low = da - (double)high * 4294967296.f; // high * 0x1p32f; + return ((du_int)high << 32) | low; } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunssfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunssfdi(fp_t a) { return __fixuint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { - return __fixunssfdi(a); -} +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { return __fixunssfdi(a); } #else -AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunssfdi); +COMPILER_RT_ALIAS(__fixunssfdi, __aeabi_f2ulz) #endif #endif diff --git a/lib/builtins/fixunssfsi.c b/lib/builtins/fixunssfsi.c index eca2916a5..738c1bb95 100644 --- a/lib/builtins/fixunssfsi.c +++ b/lib/builtins/fixunssfsi.c @@ -1,33 +1,26 @@ -/* ===-- fixunssfsi.c - Implement __fixunssfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunssfsi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfsi.c - Implement __fixunssfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunssfsi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunssfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunssfsi(fp_t a) { return __fixuint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { - return __fixunssfsi(a); -} +AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { return __fixunssfsi(a); } #else -AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunssfsi); +COMPILER_RT_ALIAS(__fixunssfsi, __aeabi_f2uiz) #endif #endif diff --git a/lib/builtins/fixunssfti.c b/lib/builtins/fixunssfti.c index 862d7bd6c..5525d77f2 100644 --- a/lib/builtins/fixunssfti.c +++ b/lib/builtins/fixunssfti.c @@ -1,16 +1,14 @@ -/* ===-- fixunssfti.c - Implement __fixunssfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunssfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfti.c - Implement __fixunssfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunssfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" @@ -19,8 +17,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunssfti(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI tu_int __fixunssfti(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfdi.c b/lib/builtins/fixunstfdi.c index b2995f658..a0805e63d 100644 --- a/lib/builtins/fixunstfdi.c +++ b/lib/builtins/fixunstfdi.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfdi.c - Implement __fixunstfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfdi.c - Implement __fixunstfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunstfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunstfdi(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfsi.c b/lib/builtins/fixunstfsi.c index b5d3f6a7d..3a1320ed3 100644 --- a/lib/builtins/fixunstfsi.c +++ b/lib/builtins/fixunstfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunstfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunstfsi(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfti.c b/lib/builtins/fixunstfti.c index 22ff9dfc0..23cd1ab61 100644 --- a/lib/builtins/fixunstfti.c +++ b/lib/builtins/fixunstfti.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunstfti(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI tu_int __fixunstfti(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunsxfdi.c b/lib/builtins/fixunsxfdi.c index 075304e78..75c4f0937 100644 --- a/lib/builtins/fixunsxfdi.c +++ b/lib/builtins/fixunsxfdi.c @@ -1,46 +1,39 @@ -/* ===-- fixunsxfdi.c - Implement __fixunsxfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfdi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfdi.c - Implement __fixunsxfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfdi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a unsigned long long, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned long long, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * du_int is a 64 bit integral type - * value in long double is representable in du_int or is negative - * (no range checking performed) - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes du_int is a 64 bit integral type value in long double is representable +// in du_int or is negative (no range checking performed) -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI du_int -__fixunsxfdi(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(du_int) * CHAR_BIT) - return ~(du_int)0; - return fb.u.low.all >> (63 - e); +COMPILER_RT_ABI du_int __fixunsxfdi(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(du_int) * CHAR_BIT) + return ~(du_int)0; + return fb.u.low.all >> (63 - e); } #endif diff --git a/lib/builtins/fixunsxfsi.c b/lib/builtins/fixunsxfsi.c index c3c70f743..1432d8ba9 100644 --- a/lib/builtins/fixunsxfsi.c +++ b/lib/builtins/fixunsxfsi.c @@ -1,45 +1,39 @@ -/* ===-- fixunsxfsi.c - Implement __fixunsxfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfsi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfsi.c - Implement __fixunsxfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfsi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a unsigned int, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned int, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * su_int is a 32 bit integral type - * value in long double is representable in su_int or is negative - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes su_int is a 32 bit integral type value in long double is representable +// in su_int or is negative -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI su_int -__fixunsxfsi(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(su_int) * CHAR_BIT) - return ~(su_int)0; - return fb.u.low.s.high >> (31 - e); +COMPILER_RT_ABI su_int __fixunsxfsi(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(su_int) * CHAR_BIT) + return ~(su_int)0; + return fb.u.low.s.high >> (31 - e); } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/fixunsxfti.c b/lib/builtins/fixunsxfti.c index fb39d00ff..508554e4f 100644 --- a/lib/builtins/fixunsxfti.c +++ b/lib/builtins/fixunsxfti.c @@ -1,50 +1,44 @@ -/* ===-- fixunsxfti.c - Implement __fixunsxfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfti.c - Implement __fixunsxfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a unsigned long long, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned long long, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * tu_int is a 128 bit integral type - * value in long double is representable in tu_int or is negative - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes tu_int is a 128 bit integral type value in long double is representable +// in tu_int or is negative -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI tu_int -__fixunsxfti(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(tu_int) * CHAR_BIT) - return ~(tu_int)0; - tu_int r = fb.u.low.all; - if (e > 63) - r <<= (e - 63); - else - r >>= (63 - e); - return r; +COMPILER_RT_ABI tu_int __fixunsxfti(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(tu_int) * CHAR_BIT) + return ~(tu_int)0; + tu_int r = fb.u.low.all; + if (e > 63) + r <<= (e - 63); + else + r >>= (63 - e); + return r; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixxfdi.c b/lib/builtins/fixxfdi.c index 011787f9e..4783c0101 100644 --- a/lib/builtins/fixxfdi.c +++ b/lib/builtins/fixxfdi.c @@ -1,48 +1,43 @@ -/* ===-- fixxfdi.c - Implement __fixxfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixxfdi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixxfdi.c - Implement __fixxfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixxfdi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a signed long long, rounding toward zero. */ +// Returns: convert a to a signed long long, rounding toward zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * di_int is a 64 bit integral type - * value in long double is representable in di_int (no range checking performed) - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes di_int is a 64 bit integral type value in long double is representable +// in di_int (no range checking performed) -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI di_int -__fixxfdi(long double a) -{ - const di_int di_max = (di_int)((~(du_int)0) / 2); - const di_int di_min = -di_max - 1; - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0) - return 0; - if ((unsigned)e >= sizeof(di_int) * CHAR_BIT) - return a > 0 ? di_max : di_min; - di_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); - di_int r = fb.u.low.all; - r = (du_int)r >> (63 - e); - return (r ^ s) - s; +COMPILER_RT_ABI di_int __fixxfdi(long double a) { + const di_int di_max = (di_int)((~(du_int)0) / 2); + const di_int di_min = -di_max - 1; + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0) + return 0; + if ((unsigned)e >= sizeof(di_int) * CHAR_BIT) + return a > 0 ? di_max : di_min; + di_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); + di_int r = fb.u.low.all; + r = (du_int)r >> (63 - e); + return (r ^ s) - s; } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/fixxfti.c b/lib/builtins/fixxfti.c index 968a4f0d5..90e03116e 100644 --- a/lib/builtins/fixxfti.c +++ b/lib/builtins/fixxfti.c @@ -1,51 +1,46 @@ -/* ===-- fixxfti.c - Implement __fixxfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixxfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixxfti.c - Implement __fixxfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixxfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a signed long long, rounding toward zero. */ +// Returns: convert a to a signed long long, rounding toward zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * ti_int is a 128 bit integral type - * value in long double is representable in ti_int - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes ti_int is a 128 bit integral type value in long double is representable +// in ti_int -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI ti_int -__fixxfti(long double a) -{ - const ti_int ti_max = (ti_int)((~(tu_int)0) / 2); - const ti_int ti_min = -ti_max - 1; - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0) - return 0; - ti_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); - ti_int r = fb.u.low.all; - if ((unsigned)e >= sizeof(ti_int) * CHAR_BIT) - return a > 0 ? ti_max : ti_min; - if (e > 63) - r <<= (e - 63); - else - r >>= (63 - e); - return (r ^ s) - s; +COMPILER_RT_ABI ti_int __fixxfti(long double a) { + const ti_int ti_max = (ti_int)((~(tu_int)0) / 2); + const ti_int ti_min = -ti_max - 1; + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0) + return 0; + ti_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); + ti_int r = fb.u.low.all; + if ((unsigned)e >= sizeof(ti_int) * CHAR_BIT) + return a > 0 ? ti_max : ti_min; + if (e > 63) + r <<= (e - 63); + else + r >>= (63 - e); + return (r ^ s) - s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatdidf.c b/lib/builtins/floatdidf.c index 36b856e07..8f887314b 100644 --- a/lib/builtins/floatdidf.c +++ b/lib/builtins/floatdidf.c @@ -1,115 +1,103 @@ -/*===-- floatdidf.c - Implement __floatdidf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __floatdidf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatdidf.c - Implement __floatdidf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * di_int is a 64 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// di_int is a 64 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; we'll set the inexact flag - * as a side-effect of this computation. - */ +// Support for systems that have hardware floating-point; we'll set the inexact +// flag as a side-effect of this computation. -COMPILER_RT_ABI double -__floatdidf(di_int a) -{ - static const double twop52 = 4503599627370496.0; // 0x1.0p52 - static const double twop32 = 4294967296.0; // 0x1.0p32 +COMPILER_RT_ABI double __floatdidf(di_int a) { + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop32 = 4294967296.0; // 0x1.0p32 - union { int64_t x; double d; } low = { .d = twop52 }; + union { + int64_t x; + double d; + } low = {.d = twop52}; - const double high = (int32_t)(a >> 32) * twop32; - low.x |= a & INT64_C(0x00000000ffffffff); + const double high = (int32_t)(a >> 32) * twop32; + low.x |= a & INT64_C(0x00000000ffffffff); - const double result = (high - twop52) + low.d; - return result; + const double result = (high - twop52) + low.d; + return result; } #else -/* Support for systems that don't have hardware floating-point; there are no flags to - * set, and we don't want to code-gen to an unknown soft-float implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. -COMPILER_RT_ABI double -__floatdidf(di_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = ((du_int)a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((du_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatdidf(di_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = ((du_int)a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((du_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((su_int)s & 0x80000000) | /* sign */ - ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((su_int)s & 0x80000000) | // sign + ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_l2d(di_int a) { - return __floatdidf(a); -} +AEABI_RTABI double __aeabi_l2d(di_int a) { return __floatdidf(a); } #else -AEABI_RTABI double __aeabi_l2d(di_int a) COMPILER_RT_ALIAS(__floatdidf); +COMPILER_RT_ALIAS(__floatdidf, __aeabi_l2d) #endif #endif diff --git a/lib/builtins/floatdisf.c b/lib/builtins/floatdisf.c index a2f09eb2e..cd9e0a3b7 100644 --- a/lib/builtins/floatdisf.c +++ b/lib/builtins/floatdisf.c @@ -1,88 +1,75 @@ -/*===-- floatdisf.c - Implement __floatdisf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __floatdisf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatdisf.c - Implement __floatdisf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a float, rounding toward even.*/ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * di_int is a 64 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// di_int is a 64 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm #include "int_lib.h" -COMPILER_RT_ABI float -__floatdisf(di_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = ((du_int)a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((du_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatdisf(di_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = ((du_int)a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((du_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((su_int)s & 0x80000000) | /* sign */ - ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((su_int)s & 0x80000000) | // sign + ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_l2f(di_int a) { - return __floatdisf(a); -} +AEABI_RTABI float __aeabi_l2f(di_int a) { return __floatdisf(a); } #else -AEABI_RTABI float __aeabi_l2f(di_int a) COMPILER_RT_ALIAS(__floatdisf); +COMPILER_RT_ALIAS(__floatdisf, __aeabi_l2f) #endif #endif diff --git a/lib/builtins/floatditf.c b/lib/builtins/floatditf.c index cd51dd8aa..9b07b6582 100644 --- a/lib/builtins/floatditf.c +++ b/lib/builtins/floatditf.c @@ -1,9 +1,8 @@ //===-- lib/floatditf.c - integer -> quad-precision conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,32 +18,32 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatditf(di_int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - du_int aAbs = (du_int)a; - if (a < 0) { - sign = signBit; - aAbs = ~(du_int)a + 1U; - } + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + du_int aAbs = (du_int)a; + if (a < 0) { + sign = signBit; + aAbs = ~(du_int)a + 1U; + } - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clzll(aAbs); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(aAbs); + rep_t result; - // Shift a into the significand field, rounding if it is a right-shift - const int shift = significandBits - exponent; - result = (rep_t)aAbs << shift ^ implicitBit; + // Shift a into the significand field, rounding if it is a right-shift + const int shift = significandBits - exponent; + result = (rep_t)aAbs << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #endif diff --git a/lib/builtins/floatdixf.c b/lib/builtins/floatdixf.c index d39e81d7c..ad5deb2d4 100644 --- a/lib/builtins/floatdixf.c +++ b/lib/builtins/floatdixf.c @@ -1,46 +1,41 @@ -/* ===-- floatdixf.c - Implement __floatdixf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatdixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatdixf.c - Implement __floatdixf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * di_int is a 64 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits di_int is a 64 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floatdixf(di_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int clz = __builtin_clzll(a); - int e = (N - 1) - clz ; /* exponent */ - long_double_bits fb; - fb.u.high.s.low = ((su_int)s & 0x00008000) | /* sign */ - (e + 16383); /* exponent */ - fb.u.low.all = a << clz; /* mantissa */ - return fb.f; +COMPILER_RT_ABI long double __floatdixf(di_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int clz = __builtin_clzll(a); + int e = (N - 1) - clz; // exponent + long_double_bits fb; + fb.u.high.s.low = ((su_int)s & 0x00008000) | // sign + (e + 16383); // exponent + fb.u.low.all = a << clz; // mantissa + return fb.f; } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/floatsidf.c b/lib/builtins/floatsidf.c index fe051123c..2c66167d7 100644 --- a/lib/builtins/floatsidf.c +++ b/lib/builtins/floatsidf.c @@ -1,9 +1,8 @@ //===-- lib/floatsidf.c - integer -> double-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,44 +17,41 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatsidf(int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); - - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - if (a < 0) { - sign = signBit; - a = -a; - } - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field and clear the implicit bit. Extra - // cast to unsigned int is necessary to get the correct behavior for - // the input INT_MIN. - const int shift = significandBits - exponent; - result = (rep_t)(unsigned int)a << shift ^ implicitBit; - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); +COMPILER_RT_ABI fp_t __floatsidf(int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + if (a < 0) { + sign = signBit; + a = -a; + } + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field and clear the implicit bit. Extra + // cast to unsigned int is necessary to get the correct behavior for + // the input INT_MIN. + const int shift = significandBits - exponent; + result = (rep_t)(unsigned int)a << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_i2d(int a) { - return __floatsidf(a); -} +AEABI_RTABI fp_t __aeabi_i2d(int a) { return __floatsidf(a); } #else -AEABI_RTABI fp_t __aeabi_i2d(int a) COMPILER_RT_ALIAS(__floatsidf); +COMPILER_RT_ALIAS(__floatsidf, __aeabi_i2d) #endif #endif diff --git a/lib/builtins/floatsisf.c b/lib/builtins/floatsisf.c index bf087ee3c..fe0604077 100644 --- a/lib/builtins/floatsisf.c +++ b/lib/builtins/floatsisf.c @@ -1,9 +1,8 @@ //===-- lib/floatsisf.c - integer -> single-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,50 +17,49 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatsisf(int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); - - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - if (a < 0) { - sign = signBit; - a = -a; - } - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field, rounding if it is a right-shift - if (exponent <= significandBits) { - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - } else { - const int shift = exponent - significandBits; - result = (rep_t)a >> shift ^ implicitBit; - rep_t round = (rep_t)a << (typeWidth - shift); - if (round > signBit) result++; - if (round == signBit) result += result & 1; - } - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); +COMPILER_RT_ABI fp_t __floatsisf(int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + if (a < 0) { + sign = signBit; + a = -a; + } + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field, rounding if it is a right-shift + if (exponent <= significandBits) { + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + } else { + const int shift = exponent - significandBits; + result = (rep_t)a >> shift ^ implicitBit; + rep_t round = (rep_t)a << (typeWidth - shift); + if (round > signBit) + result++; + if (round == signBit) + result += result & 1; + } + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_i2f(int a) { - return __floatsisf(a); -} +AEABI_RTABI fp_t __aeabi_i2f(int a) { return __floatsisf(a); } #else -AEABI_RTABI fp_t __aeabi_i2f(int a) COMPILER_RT_ALIAS(__floatsisf); +COMPILER_RT_ALIAS(__floatsisf, __aeabi_i2f) #endif #endif diff --git a/lib/builtins/floatsitf.c b/lib/builtins/floatsitf.c index f0abca363..f56063f36 100644 --- a/lib/builtins/floatsitf.c +++ b/lib/builtins/floatsitf.c @@ -1,9 +1,8 @@ //===-- lib/floatsitf.c - integer -> quad-precision conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,32 +18,32 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatsitf(int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - unsigned aAbs = (unsigned)a; - if (a < 0) { - sign = signBit; - aAbs = ~(unsigned)a + 1U; - } + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + unsigned aAbs = (unsigned)a; + if (a < 0) { + sign = signBit; + aAbs = ~(unsigned)a + 1U; + } - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(aAbs); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(aAbs); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)aAbs << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)aAbs << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #endif diff --git a/lib/builtins/floattidf.c b/lib/builtins/floattidf.c index 2702a3c8a..0a1c04bec 100644 --- a/lib/builtins/floattidf.c +++ b/lib/builtins/floattidf.c @@ -1,83 +1,73 @@ -/* ===-- floattidf.c - Implement __floattidf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattidf.c - Implement __floattidf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a double, rounding toward even.*/ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm -COMPILER_RT_ABI double -__floattidf(ti_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floattidf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((su_int)s & 0x80000000) | /* sign */ - ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((su_int)s & 0x80000000) | // sign + ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floattisf.c b/lib/builtins/floattisf.c index f1b585f2c..a8fcdbe14 100644 --- a/lib/builtins/floattisf.c +++ b/lib/builtins/floattisf.c @@ -1,82 +1,71 @@ -/* ===-- floattisf.c - Implement __floattisf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattisf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattisf.c - Implement __floattisf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm -COMPILER_RT_ABI float -__floattisf(ti_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floattisf(ti_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((tu_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((su_int)s & 0x80000000) | /* sign */ - ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((su_int)s & 0x80000000) | // sign + ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floattitf.c b/lib/builtins/floattitf.c index 994fded39..196cbdae1 100644 --- a/lib/builtins/floattitf.c +++ b/lib/builtins/floattitf.c @@ -1,9 +1,8 @@ //===-- lib/floattitf.c - int128 -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,66 +16,63 @@ #include "fp_lib.h" #include "int_lib.h" -/* Returns: convert a ti_int to a fp_t, rounding toward even. */ +// Returns: convert a ti_int to a fp_t, rounding toward even. -/* Assumption: fp_t is a IEEE 128 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: fp_t is a IEEE 128 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | - * mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -COMPILER_RT_ABI fp_t -__floattitf(ti_int a) { - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ - } else { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI fp_t __floattitf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } - long_double_bits fb; - fb.u.high.all = (s & 0x8000000000000000LL) /* sign */ - | (du_int)(e + 16383) << 48 /* exponent */ - | ((a >> 64) & 0x0000ffffffffffffLL); /* significand */ - fb.u.low.all = (du_int)(a); - return fb.f; + long_double_bits fb; + fb.u.high.all = (s & 0x8000000000000000LL) // sign + | (du_int)(e + 16383) << 48 // exponent + | ((a >> 64) & 0x0000ffffffffffffLL); // significand + fb.u.low.all = (du_int)(a); + return fb.f; } #endif diff --git a/lib/builtins/floattixf.c b/lib/builtins/floattixf.c index 1203b3a96..23796f1bb 100644 --- a/lib/builtins/floattixf.c +++ b/lib/builtins/floattixf.c @@ -1,84 +1,73 @@ -/* ===-- floattixf.c - Implement __floattixf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattixf.c - Implement __floattixf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * ti_int is a 128 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits ti_int is a 128 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floattixf(ti_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI long double __floattixf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ - } - long_double_bits fb; - fb.u.high.s.low = ((su_int)s & 0x8000) | /* sign */ - (e + 16383); /* exponent */ - fb.u.low.all = (du_int)a; /* mantissa */ - return fb.f; + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } + long_double_bits fb; + fb.u.high.s.low = ((su_int)s & 0x8000) | // sign + (e + 16383); // exponent + fb.u.low.all = (du_int)a; // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatundidf.c b/lib/builtins/floatundidf.c index 8bc2a0963..e7c6aae5c 100644 --- a/lib/builtins/floatundidf.c +++ b/lib/builtins/floatundidf.c @@ -1,114 +1,106 @@ -/* ===-- floatundidf.c - Implement __floatundidf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatundidf.c - Implement __floatundidf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * du_int is a 64 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// du_int is a 64 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm #include "int_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; we'll set the inexact flag - * as a side-effect of this computation. - */ +// Support for systems that have hardware floating-point; we'll set the inexact +// flag as a side-effect of this computation. -COMPILER_RT_ABI double -__floatundidf(du_int a) -{ - static const double twop52 = 4503599627370496.0; // 0x1.0p52 - static const double twop84 = 19342813113834066795298816.0; // 0x1.0p84 - static const double twop84_plus_twop52 = 19342813118337666422669312.0; // 0x1.00000001p84 +COMPILER_RT_ABI double __floatundidf(du_int a) { + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop84 = 19342813113834066795298816.0; // 0x1.0p84 + static const double twop84_plus_twop52 = + 19342813118337666422669312.0; // 0x1.00000001p84 - union { uint64_t x; double d; } high = { .d = twop84 }; - union { uint64_t x; double d; } low = { .d = twop52 }; + union { + uint64_t x; + double d; + } high = {.d = twop84}; + union { + uint64_t x; + double d; + } low = {.d = twop52}; - high.x |= a >> 32; - low.x |= a & UINT64_C(0x00000000ffffffff); + high.x |= a >> 32; + low.x |= a & UINT64_C(0x00000000ffffffff); - const double result = (high.d - twop84_plus_twop52) + low.d; - return result; + const double result = (high.d - twop84_plus_twop52) + low.d; + return result; } #else -/* Support for systems that don't have hardware floating-point; there are no flags to - * set, and we don't want to code-gen to an unknown soft-float implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. -COMPILER_RT_ABI double -__floatundidf(du_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((du_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatundidf(du_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((du_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_ul2d(du_int a) { - return __floatundidf(a); -} +AEABI_RTABI double __aeabi_ul2d(du_int a) { return __floatundidf(a); } #else -AEABI_RTABI double __aeabi_ul2d(du_int a) COMPILER_RT_ALIAS(__floatundidf); +COMPILER_RT_ALIAS(__floatundidf, __aeabi_ul2d) #endif #endif diff --git a/lib/builtins/floatundisf.c b/lib/builtins/floatundisf.c index 844786ea7..87841b761 100644 --- a/lib/builtins/floatundisf.c +++ b/lib/builtins/floatundisf.c @@ -1,85 +1,72 @@ -/*===-- floatundisf.c - Implement __floatundisf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundisf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatundisf.c - Implement __floatundisf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * du_int is a 64 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// du_int is a 64 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm #include "int_lib.h" -COMPILER_RT_ABI float -__floatundisf(du_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* 8 exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((du_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatundisf(du_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // 8 exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((du_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_ul2f(du_int a) { - return __floatundisf(a); -} +AEABI_RTABI float __aeabi_ul2f(du_int a) { return __floatundisf(a); } #else -AEABI_RTABI float __aeabi_ul2f(du_int a) COMPILER_RT_ALIAS(__floatundisf); +COMPILER_RT_ALIAS(__floatundisf, __aeabi_ul2f) #endif #endif diff --git a/lib/builtins/floatunditf.c b/lib/builtins/floatunditf.c index 8098e95e8..8d310851e 100644 --- a/lib/builtins/floatunditf.c +++ b/lib/builtins/floatunditf.c @@ -1,9 +1,8 @@ //===-- lib/floatunditf.c - uint -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,22 +18,23 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatunditf(du_int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clzll(a); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(a); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #endif diff --git a/lib/builtins/floatundixf.c b/lib/builtins/floatundixf.c index ca5e06d64..85264adac 100644 --- a/lib/builtins/floatundixf.c +++ b/lib/builtins/floatundixf.c @@ -1,42 +1,37 @@ -/* ===-- floatundixf.c - Implement __floatundixf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatundixf.c - Implement __floatundixf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * du_int is a 64 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits du_int is a 64 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ -COMPILER_RT_ABI long double -__floatundixf(du_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int clz = __builtin_clzll(a); - int e = (N - 1) - clz ; /* exponent */ - long_double_bits fb; - fb.u.high.s.low = (e + 16383); /* exponent */ - fb.u.low.all = a << clz; /* mantissa */ - return fb.f; +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm +COMPILER_RT_ABI long double __floatundixf(du_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int clz = __builtin_clzll(a); + int e = (N - 1) - clz; // exponent + long_double_bits fb; + fb.u.high.s.low = (e + 16383); // exponent + fb.u.low.all = a << clz; // mantissa + return fb.f; } -#endif /* _ARCH_PPC */ +#endif // _ARCH_PPC diff --git a/lib/builtins/floatunsidf.c b/lib/builtins/floatunsidf.c index 75cf6b917..2c01c3041 100644 --- a/lib/builtins/floatunsidf.c +++ b/lib/builtins/floatunsidf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsidf.c - uint -> double-precision conversion ---*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,33 +17,31 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatunsidf(unsigned int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); +COMPILER_RT_ABI fp_t __floatunsidf(unsigned int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { - return __floatunsidf(a); -} +AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { return __floatunsidf(a); } #else -AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) COMPILER_RT_ALIAS(__floatunsidf); +COMPILER_RT_ALIAS(__floatunsidf, __aeabi_ui2d) #endif #endif diff --git a/lib/builtins/floatunsisf.c b/lib/builtins/floatunsisf.c index 29525cced..33a1b5ae2 100644 --- a/lib/builtins/floatunsisf.c +++ b/lib/builtins/floatunsisf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsisf.c - uint -> single-precision conversion ---*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,41 +17,41 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatunsisf(unsigned int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field, rounding if it is a right-shift - if (exponent <= significandBits) { - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - } else { - const int shift = exponent - significandBits; - result = (rep_t)a >> shift ^ implicitBit; - rep_t round = (rep_t)a << (typeWidth - shift); - if (round > signBit) result++; - if (round == signBit) result += result & 1; - } - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); +COMPILER_RT_ABI fp_t __floatunsisf(unsigned int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field, rounding if it is a right-shift + if (exponent <= significandBits) { + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + } else { + const int shift = exponent - significandBits; + result = (rep_t)a >> shift ^ implicitBit; + rep_t round = (rep_t)a << (typeWidth - shift); + if (round > signBit) + result++; + if (round == signBit) + result += result & 1; + } + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { - return __floatunsisf(a); -} +AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { return __floatunsisf(a); } #else -AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) COMPILER_RT_ALIAS(__floatunsisf); +COMPILER_RT_ALIAS(__floatunsisf, __aeabi_ui2f) #endif #endif diff --git a/lib/builtins/floatunsitf.c b/lib/builtins/floatunsitf.c index 1cd1842e7..a4bf0f65f 100644 --- a/lib/builtins/floatunsitf.c +++ b/lib/builtins/floatunsitf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsitf.c - uint -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,22 +18,23 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatunsitf(unsigned int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #endif diff --git a/lib/builtins/floatuntidf.c b/lib/builtins/floatuntidf.c index 960265d80..e69e65c1a 100644 --- a/lib/builtins/floatuntidf.c +++ b/lib/builtins/floatuntidf.c @@ -1,80 +1,70 @@ -/* ===-- floatuntidf.c - Implement __floatuntidf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntidf.c - Implement __floatuntidf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm -COMPILER_RT_ABI double -__floatuntidf(tu_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatuntidf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatuntisf.c b/lib/builtins/floatuntisf.c index c0dd0275d..9dec0ab5c 100644 --- a/lib/builtins/floatuntisf.c +++ b/lib/builtins/floatuntisf.c @@ -1,79 +1,68 @@ -/* ===-- floatuntisf.c - Implement __floatuntisf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntisf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntisf.c - Implement __floatuntisf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm -COMPILER_RT_ABI float -__floatuntisf(tu_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatuntisf(tu_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((tu_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatuntitf.c b/lib/builtins/floatuntitf.c index e2518c93f..d308d3118 100644 --- a/lib/builtins/floatuntitf.c +++ b/lib/builtins/floatuntitf.c @@ -1,9 +1,8 @@ //===-- lib/floatuntitf.c - uint128 -> quad-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,63 +16,60 @@ #include "fp_lib.h" #include "int_lib.h" -/* Returns: convert a tu_int to a fp_t, rounding toward even. */ +// Returns: convert a tu_int to a fp_t, rounding toward even. -/* Assumption: fp_t is a IEEE 128 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: fp_t is a IEEE 128 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | - * mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -COMPILER_RT_ABI fp_t -__floatuntitf(tu_int a) { - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ - } else { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } - long_double_bits fb; - fb.u.high.all = (du_int)(e + 16383) << 48 /* exponent */ - | ((a >> 64) & 0x0000ffffffffffffLL); /* significand */ - fb.u.low.all = (du_int)(a); - return fb.f; + long_double_bits fb; + fb.u.high.all = (du_int)(e + 16383) << 48 // exponent + | ((a >> 64) & 0x0000ffffffffffffLL); // significand + fb.u.low.all = (du_int)(a); + return fb.f; } #endif diff --git a/lib/builtins/floatuntixf.c b/lib/builtins/floatuntixf.c index ea81cb1bc..efd8a27a0 100644 --- a/lib/builtins/floatuntixf.c +++ b/lib/builtins/floatuntixf.c @@ -1,81 +1,70 @@ -/* ===-- floatuntixf.c - Implement __floatuntixf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntixf.c - Implement __floatuntixf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * tu_int is a 128 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits tu_int is a 128 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floatuntixf(tu_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI long double __floatuntixf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ - } - long_double_bits fb; - fb.u.high.s.low = (e + 16383); /* exponent */ - fb.u.low.all = (du_int)a; /* mantissa */ - return fb.f; + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } + long_double_bits fb; + fb.u.high.s.low = (e + 16383); // exponent + fb.u.low.all = (du_int)a; // mantissa + return fb.f; } #endif diff --git a/lib/builtins/fp_add_impl.inc b/lib/builtins/fp_add_impl.inc index b47be1b64..b5a2fb098 100644 --- a/lib/builtins/fp_add_impl.inc +++ b/lib/builtins/fp_add_impl.inc @@ -1,9 +1,8 @@ //===----- lib/fp_add_impl.inc - floaing point addition -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,130 +14,143 @@ #include "fp_lib.h" static __inline fp_t __addXf3__(fp_t a, fp_t b) { - rep_t aRep = toRep(a); - rep_t bRep = toRep(b); - const rep_t aAbs = aRep & absMask; - const rep_t bAbs = bRep & absMask; - - // Detect if a or b is zero, infinity, or NaN. - if (aAbs - REP_C(1) >= infRep - REP_C(1) || - bAbs - REP_C(1) >= infRep - REP_C(1)) { - // NaN + anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything + NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // +/-infinity + -/+infinity = qNaN - if ((toRep(a) ^ toRep(b)) == signBit) return fromRep(qnanRep); - // +/-infinity + anything remaining = +/- infinity - else return a; - } - - // anything remaining + +/-infinity = +/-infinity - if (bAbs == infRep) return b; - - // zero + anything = anything - if (!aAbs) { - // but we need to get the sign right for zero + zero - if (!bAbs) return fromRep(toRep(a) & toRep(b)); - else return b; - } - - // anything + zero = anything - if (!bAbs) return a; + rep_t aRep = toRep(a); + rep_t bRep = toRep(b); + const rep_t aAbs = aRep & absMask; + const rep_t bAbs = bRep & absMask; + + // Detect if a or b is zero, infinity, or NaN. + if (aAbs - REP_C(1) >= infRep - REP_C(1) || + bAbs - REP_C(1) >= infRep - REP_C(1)) { + // NaN + anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything + NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // +/-infinity + -/+infinity = qNaN + if ((toRep(a) ^ toRep(b)) == signBit) + return fromRep(qnanRep); + // +/-infinity + anything remaining = +/- infinity + else + return a; } - // Swap a and b if necessary so that a has the larger absolute value. - if (bAbs > aAbs) { - const rep_t temp = aRep; - aRep = bRep; - bRep = temp; + // anything remaining + +/-infinity = +/-infinity + if (bAbs == infRep) + return b; + + // zero + anything = anything + if (!aAbs) { + // but we need to get the sign right for zero + zero + if (!bAbs) + return fromRep(toRep(a) & toRep(b)); + else + return b; } - // Extract the exponent and significand from the (possibly swapped) a and b. - int aExponent = aRep >> significandBits & maxExponent; - int bExponent = bRep >> significandBits & maxExponent; - rep_t aSignificand = aRep & significandMask; - rep_t bSignificand = bRep & significandMask; - - // Normalize any denormals, and adjust the exponent accordingly. - if (aExponent == 0) aExponent = normalize(&aSignificand); - if (bExponent == 0) bExponent = normalize(&bSignificand); - - // The sign of the result is the sign of the larger operand, a. If they - // have opposite signs, we are performing a subtraction; otherwise addition. - const rep_t resultSign = aRep & signBit; - const bool subtraction = (aRep ^ bRep) & signBit; - - // Shift the significands to give us round, guard and sticky, and or in the - // implicit significand bit. (If we fell through from the denormal path it - // was already set by normalize( ), but setting it twice won't hurt - // anything.) - aSignificand = (aSignificand | implicitBit) << 3; - bSignificand = (bSignificand | implicitBit) << 3; - - // Shift the significand of b by the difference in exponents, with a sticky - // bottom bit to get rounding correct. - const unsigned int align = aExponent - bExponent; - if (align) { - if (align < typeWidth) { - const bool sticky = bSignificand << (typeWidth - align); - bSignificand = bSignificand >> align | sticky; - } else { - bSignificand = 1; // sticky; b is known to be non-zero. - } - } - if (subtraction) { - aSignificand -= bSignificand; - // If a == -b, return +zero. - if (aSignificand == 0) return fromRep(0); - - // If partial cancellation occured, we need to left-shift the result - // and adjust the exponent: - if (aSignificand < implicitBit << 3) { - const int shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); - aSignificand <<= shift; - aExponent -= shift; - } + // anything + zero = anything + if (!bAbs) + return a; + } + + // Swap a and b if necessary so that a has the larger absolute value. + if (bAbs > aAbs) { + const rep_t temp = aRep; + aRep = bRep; + bRep = temp; + } + + // Extract the exponent and significand from the (possibly swapped) a and b. + int aExponent = aRep >> significandBits & maxExponent; + int bExponent = bRep >> significandBits & maxExponent; + rep_t aSignificand = aRep & significandMask; + rep_t bSignificand = bRep & significandMask; + + // Normalize any denormals, and adjust the exponent accordingly. + if (aExponent == 0) + aExponent = normalize(&aSignificand); + if (bExponent == 0) + bExponent = normalize(&bSignificand); + + // The sign of the result is the sign of the larger operand, a. If they + // have opposite signs, we are performing a subtraction; otherwise addition. + const rep_t resultSign = aRep & signBit; + const bool subtraction = (aRep ^ bRep) & signBit; + + // Shift the significands to give us round, guard and sticky, and or in the + // implicit significand bit. (If we fell through from the denormal path it + // was already set by normalize( ), but setting it twice won't hurt + // anything.) + aSignificand = (aSignificand | implicitBit) << 3; + bSignificand = (bSignificand | implicitBit) << 3; + + // Shift the significand of b by the difference in exponents, with a sticky + // bottom bit to get rounding correct. + const unsigned int align = aExponent - bExponent; + if (align) { + if (align < typeWidth) { + const bool sticky = bSignificand << (typeWidth - align); + bSignificand = bSignificand >> align | sticky; + } else { + bSignificand = 1; // sticky; b is known to be non-zero. } - else /* addition */ { - aSignificand += bSignificand; - - // If the addition carried up, we need to right-shift the result and - // adjust the exponent: - if (aSignificand & implicitBit << 4) { - const bool sticky = aSignificand & 1; - aSignificand = aSignificand >> 1 | sticky; - aExponent += 1; - } + } + if (subtraction) { + aSignificand -= bSignificand; + // If a == -b, return +zero. + if (aSignificand == 0) + return fromRep(0); + + // If partial cancellation occured, we need to left-shift the result + // and adjust the exponent: + if (aSignificand < implicitBit << 3) { + const int shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); + aSignificand <<= shift; + aExponent -= shift; } - - // If we have overflowed the type, return +/- infinity: - if (aExponent >= maxExponent) return fromRep(infRep | resultSign); - - if (aExponent <= 0) { - // Result is denormal before rounding; the exponent is zero and we - // need to shift the significand. - const int shift = 1 - aExponent; - const bool sticky = aSignificand << (typeWidth - shift); - aSignificand = aSignificand >> shift | sticky; - aExponent = 0; + } else /* addition */ { + aSignificand += bSignificand; + + // If the addition carried up, we need to right-shift the result and + // adjust the exponent: + if (aSignificand & implicitBit << 4) { + const bool sticky = aSignificand & 1; + aSignificand = aSignificand >> 1 | sticky; + aExponent += 1; } - - // Low three bits are round, guard, and sticky. - const int roundGuardSticky = aSignificand & 0x7; - - // Shift the significand into place, and mask off the implicit bit. - rep_t result = aSignificand >> 3 & significandMask; - - // Insert the exponent and sign. - result |= (rep_t)aExponent << significandBits; - result |= resultSign; - - // Final rounding. The result may overflow to infinity, but that is the - // correct result in that case. - if (roundGuardSticky > 0x4) result++; - if (roundGuardSticky == 0x4) result += result & 1; - return fromRep(result); + } + + // If we have overflowed the type, return +/- infinity: + if (aExponent >= maxExponent) + return fromRep(infRep | resultSign); + + if (aExponent <= 0) { + // Result is denormal before rounding; the exponent is zero and we + // need to shift the significand. + const int shift = 1 - aExponent; + const bool sticky = aSignificand << (typeWidth - shift); + aSignificand = aSignificand >> shift | sticky; + aExponent = 0; + } + + // Low three bits are round, guard, and sticky. + const int roundGuardSticky = aSignificand & 0x7; + + // Shift the significand into place, and mask off the implicit bit. + rep_t result = aSignificand >> 3 & significandMask; + + // Insert the exponent and sign. + result |= (rep_t)aExponent << significandBits; + result |= resultSign; + + // Final rounding. The result may overflow to infinity, but that is the + // correct result in that case. + if (roundGuardSticky > 0x4) + result++; + if (roundGuardSticky == 0x4) + result += result & 1; + return fromRep(result); } diff --git a/lib/builtins/fp_extend.h b/lib/builtins/fp_extend.h index 6d95a0680..d2083c426 100644 --- a/lib/builtins/fp_extend.h +++ b/lib/builtins/fp_extend.h @@ -1,9 +1,9 @@ -//===-lib/fp_extend.h - low precision -> high precision conversion -*- C -*-===// +//===-lib/fp_extend.h - low precision -> high precision conversion -*- C +//-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,12 +30,12 @@ typedef uint64_t src_rep_t; static const int srcSigBits = 52; static __inline int src_rep_t_clz(src_rep_t a) { #if defined __LP64__ - return __builtin_clzl(a); + return __builtin_clzl(a); #else - if (a & REP_C(0xffffffff00000000)) - return __builtin_clz(a >> 32); - else - return 32 + __builtin_clz(a & REP_C(0xffffffff)); + if (a & REP_C(0xffffffff00000000)) + return __builtin_clz(a >> 32); + else + return 32 + __builtin_clz(a & REP_C(0xffffffff)); #endif } @@ -48,7 +48,7 @@ static const int srcSigBits = 10; #else #error Source should be half, single, or double precision! -#endif //end source precision +#endif // end source precision #if defined DST_SINGLE typedef float dst_t; @@ -70,20 +70,26 @@ static const int dstSigBits = 112; #else #error Destination should be single, double, or quad precision! -#endif //end destination precision +#endif // end destination precision // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. static __inline src_rep_t srcToRep(src_t x) { - const union { src_t f; src_rep_t i; } rep = {.f = x}; - return rep.i; + const union { + src_t f; + src_rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline dst_t dstFromRep(dst_rep_t x) { - const union { dst_t f; dst_rep_t i; } rep = {.i = x}; - return rep.f; + const union { + dst_t f; + dst_rep_t i; + } rep = {.i = x}; + return rep.f; } // End helper routines. Conversion implementation follows. -#endif //FP_EXTEND_HEADER +#endif // FP_EXTEND_HEADER diff --git a/lib/builtins/fp_extend_impl.inc b/lib/builtins/fp_extend_impl.inc index b785cc768..4fa3ed8d9 100644 --- a/lib/builtins/fp_extend_impl.inc +++ b/lib/builtins/fp_extend_impl.inc @@ -1,9 +1,8 @@ //=-lib/fp_extend_impl.inc - low precision -> high precision conversion -*-- -// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,70 +38,70 @@ #include "fp_extend.h" static __inline dst_t __extendXfYf2__(src_t a) { - // Various constants whose values follow from the type parameters. - // Any reasonable optimizer will fold and propagate all of these. - const int srcBits = sizeof(src_t)*CHAR_BIT; - const int srcExpBits = srcBits - srcSigBits - 1; - const int srcInfExp = (1 << srcExpBits) - 1; - const int srcExpBias = srcInfExp >> 1; + // Various constants whose values follow from the type parameters. + // Any reasonable optimizer will fold and propagate all of these. + const int srcBits = sizeof(src_t) * CHAR_BIT; + const int srcExpBits = srcBits - srcSigBits - 1; + const int srcInfExp = (1 << srcExpBits) - 1; + const int srcExpBias = srcInfExp >> 1; - const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; - const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; - const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); - const src_rep_t srcAbsMask = srcSignMask - 1; - const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); - const src_rep_t srcNaNCode = srcQNaN - 1; + const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; + const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; + const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); + const src_rep_t srcAbsMask = srcSignMask - 1; + const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); + const src_rep_t srcNaNCode = srcQNaN - 1; - const int dstBits = sizeof(dst_t)*CHAR_BIT; - const int dstExpBits = dstBits - dstSigBits - 1; - const int dstInfExp = (1 << dstExpBits) - 1; - const int dstExpBias = dstInfExp >> 1; + const int dstBits = sizeof(dst_t) * CHAR_BIT; + const int dstExpBits = dstBits - dstSigBits - 1; + const int dstInfExp = (1 << dstExpBits) - 1; + const int dstExpBias = dstInfExp >> 1; - const dst_rep_t dstMinNormal = DST_REP_C(1) << dstSigBits; + const dst_rep_t dstMinNormal = DST_REP_C(1) << dstSigBits; - // Break a into a sign and representation of the absolute value - const src_rep_t aRep = srcToRep(a); - const src_rep_t aAbs = aRep & srcAbsMask; - const src_rep_t sign = aRep & srcSignMask; - dst_rep_t absResult; + // Break a into a sign and representation of the absolute value + const src_rep_t aRep = srcToRep(a); + const src_rep_t aAbs = aRep & srcAbsMask; + const src_rep_t sign = aRep & srcSignMask; + dst_rep_t absResult; - // If sizeof(src_rep_t) < sizeof(int), the subtraction result is promoted - // to (signed) int. To avoid that, explicitly cast to src_rep_t. - if ((src_rep_t)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) { - // a is a normal number. - // Extend to the destination type by shifting the significand and - // exponent into the proper position and rebiasing the exponent. - absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits); - absResult += (dst_rep_t)(dstExpBias - srcExpBias) << dstSigBits; - } + // If sizeof(src_rep_t) < sizeof(int), the subtraction result is promoted + // to (signed) int. To avoid that, explicitly cast to src_rep_t. + if ((src_rep_t)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) { + // a is a normal number. + // Extend to the destination type by shifting the significand and + // exponent into the proper position and rebiasing the exponent. + absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits); + absResult += (dst_rep_t)(dstExpBias - srcExpBias) << dstSigBits; + } - else if (aAbs >= srcInfinity) { - // a is NaN or infinity. - // Conjure the result by beginning with infinity, then setting the qNaN - // bit (if needed) and right-aligning the rest of the trailing NaN - // payload field. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - absResult |= (dst_rep_t)(aAbs & srcQNaN) << (dstSigBits - srcSigBits); - absResult |= (dst_rep_t)(aAbs & srcNaNCode) << (dstSigBits - srcSigBits); - } + else if (aAbs >= srcInfinity) { + // a is NaN or infinity. + // Conjure the result by beginning with infinity, then setting the qNaN + // bit (if needed) and right-aligning the rest of the trailing NaN + // payload field. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + absResult |= (dst_rep_t)(aAbs & srcQNaN) << (dstSigBits - srcSigBits); + absResult |= (dst_rep_t)(aAbs & srcNaNCode) << (dstSigBits - srcSigBits); + } - else if (aAbs) { - // a is denormal. - // renormalize the significand and clear the leading bit, then insert - // the correct adjusted exponent in the destination type. - const int scale = src_rep_t_clz(aAbs) - src_rep_t_clz(srcMinNormal); - absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits + scale); - absResult ^= dstMinNormal; - const int resultExponent = dstExpBias - srcExpBias - scale + 1; - absResult |= (dst_rep_t)resultExponent << dstSigBits; - } + else if (aAbs) { + // a is denormal. + // renormalize the significand and clear the leading bit, then insert + // the correct adjusted exponent in the destination type. + const int scale = src_rep_t_clz(aAbs) - src_rep_t_clz(srcMinNormal); + absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits + scale); + absResult ^= dstMinNormal; + const int resultExponent = dstExpBias - srcExpBias - scale + 1; + absResult |= (dst_rep_t)resultExponent << dstSigBits; + } - else { - // a is zero. - absResult = 0; - } + else { + // a is zero. + absResult = 0; + } - // Apply the signbit to (dst_t)abs(a). - const dst_rep_t result = absResult | (dst_rep_t)sign << (dstBits - srcBits); - return dstFromRep(result); + // Apply the signbit to (dst_t)abs(a). + const dst_rep_t result = absResult | (dst_rep_t)sign << (dstBits - srcBits); + return dstFromRep(result); } diff --git a/lib/builtins/fp_fixint_impl.inc b/lib/builtins/fp_fixint_impl.inc index da70d4d39..263786bdd 100644 --- a/lib/builtins/fp_fixint_impl.inc +++ b/lib/builtins/fp_fixint_impl.inc @@ -1,9 +1,8 @@ //===-- lib/fixdfsi.c - Double-precision -> integer conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,27 +14,27 @@ #include "fp_lib.h" static __inline fixint_t __fixint(fp_t a) { - const fixint_t fixint_max = (fixint_t)((~(fixuint_t)0) / 2); - const fixint_t fixint_min = -fixint_max - 1; - // Break a into sign, exponent, significand - const rep_t aRep = toRep(a); - const rep_t aAbs = aRep & absMask; - const fixint_t sign = aRep & signBit ? -1 : 1; - const int exponent = (aAbs >> significandBits) - exponentBias; - const rep_t significand = (aAbs & significandMask) | implicitBit; + const fixint_t fixint_max = (fixint_t)((~(fixuint_t)0) / 2); + const fixint_t fixint_min = -fixint_max - 1; + // Break a into sign, exponent, significand + const rep_t aRep = toRep(a); + const rep_t aAbs = aRep & absMask; + const fixint_t sign = aRep & signBit ? -1 : 1; + const int exponent = (aAbs >> significandBits) - exponentBias; + const rep_t significand = (aAbs & significandMask) | implicitBit; - // If exponent is negative, the result is zero. - if (exponent < 0) - return 0; + // If exponent is negative, the result is zero. + if (exponent < 0) + return 0; - // If the value is too large for the integer type, saturate. - if ((unsigned)exponent >= sizeof(fixint_t) * CHAR_BIT) - return sign == 1 ? fixint_max : fixint_min; + // If the value is too large for the integer type, saturate. + if ((unsigned)exponent >= sizeof(fixint_t) * CHAR_BIT) + return sign == 1 ? fixint_max : fixint_min; - // If 0 <= exponent < significandBits, right shift to get the result. - // Otherwise, shift left. - if (exponent < significandBits) - return sign * (significand >> (significandBits - exponent)); - else - return sign * ((fixint_t)significand << (exponent - significandBits)); + // If 0 <= exponent < significandBits, right shift to get the result. + // Otherwise, shift left. + if (exponent < significandBits) + return sign * (significand >> (significandBits - exponent)); + else + return sign * ((fixint_t)significand << (exponent - significandBits)); } diff --git a/lib/builtins/fp_fixuint_impl.inc b/lib/builtins/fp_fixuint_impl.inc index d68ccf27a..5fd361611 100644 --- a/lib/builtins/fp_fixuint_impl.inc +++ b/lib/builtins/fp_fixuint_impl.inc @@ -1,9 +1,8 @@ //===-- lib/fixdfsi.c - Double-precision -> integer conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,25 +14,25 @@ #include "fp_lib.h" static __inline fixuint_t __fixuint(fp_t a) { - // Break a into sign, exponent, significand - const rep_t aRep = toRep(a); - const rep_t aAbs = aRep & absMask; - const int sign = aRep & signBit ? -1 : 1; - const int exponent = (aAbs >> significandBits) - exponentBias; - const rep_t significand = (aAbs & significandMask) | implicitBit; + // Break a into sign, exponent, significand + const rep_t aRep = toRep(a); + const rep_t aAbs = aRep & absMask; + const int sign = aRep & signBit ? -1 : 1; + const int exponent = (aAbs >> significandBits) - exponentBias; + const rep_t significand = (aAbs & significandMask) | implicitBit; - // If either the value or the exponent is negative, the result is zero. - if (sign == -1 || exponent < 0) - return 0; + // If either the value or the exponent is negative, the result is zero. + if (sign == -1 || exponent < 0) + return 0; - // If the value is too large for the integer type, saturate. - if ((unsigned)exponent >= sizeof(fixuint_t) * CHAR_BIT) - return ~(fixuint_t)0; + // If the value is too large for the integer type, saturate. + if ((unsigned)exponent >= sizeof(fixuint_t) * CHAR_BIT) + return ~(fixuint_t)0; - // If 0 <= exponent < significandBits, right shift to get the result. - // Otherwise, shift left. - if (exponent < significandBits) - return significand >> (significandBits - exponent); - else - return (fixuint_t)significand << (exponent - significandBits); + // If 0 <= exponent < significandBits, right shift to get the result. + // Otherwise, shift left. + if (exponent < significandBits) + return significand >> (significandBits - exponent); + else + return (fixuint_t)significand << (exponent - significandBits); } diff --git a/lib/builtins/fp_lib.h b/lib/builtins/fp_lib.h index a0e19ab6a..83c3081aa 100644 --- a/lib/builtins/fp_lib.h +++ b/lib/builtins/fp_lib.h @@ -1,9 +1,8 @@ //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,22 +20,22 @@ #ifndef FP_LIB_HEADER #define FP_LIB_HEADER -#include <stdint.h> -#include <stdbool.h> -#include <limits.h> #include "int_lib.h" #include "int_math.h" +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in // 32-bit mode. #if defined(__FreeBSD__) && defined(__i386__) -# include <sys/param.h> -# if __FreeBSD_version < 903000 // v9.3 -# define uint64_t unsigned long long -# define int64_t long long -# undef UINT64_C -# define UINT64_C(c) (c ## ULL) -# endif +#include <sys/param.h> +#if __FreeBSD_version < 903000 // v9.3 +#define uint64_t unsigned long long +#define int64_t long long +#undef UINT64_C +#define UINT64_C(c) (c##ULL) +#endif #endif #if defined SINGLE_PRECISION @@ -47,15 +46,13 @@ typedef float fp_t; #define REP_C UINT32_C #define significandBits 23 -static __inline int rep_clz(rep_t a) { - return __builtin_clz(a); -} +static __inline int rep_clz(rep_t a) { return __builtin_clz(a); } // 32x32 --> 64 bit multiply static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product = (uint64_t)a*b; - *hi = product >> 32; - *lo = product; + const uint64_t product = (uint64_t)a * b; + *hi = product >> 32; + *lo = product; } COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); @@ -69,12 +66,12 @@ typedef double fp_t; static __inline int rep_clz(rep_t a) { #if defined __LP64__ - return __builtin_clzl(a); + return __builtin_clzl(a); #else - if (a & REP_C(0xffffffff00000000)) - return __builtin_clz(a >> 32); - else - return 32 + __builtin_clz(a & REP_C(0xffffffff)); + if (a & REP_C(0xffffffff00000000)) + return __builtin_clz(a >> 32); + else + return 32 + __builtin_clz(a & REP_C(0xffffffff)); #endif } @@ -85,17 +82,17 @@ static __inline int rep_clz(rep_t a) { // many 64-bit platforms have this operation, but they tend to have hardware // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - // Each of the component 32x32 -> 64 products - const uint64_t plolo = loWord(a) * loWord(b); - const uint64_t plohi = loWord(a) * hiWord(b); - const uint64_t philo = hiWord(a) * loWord(b); - const uint64_t phihi = hiWord(a) * hiWord(b); - // Sum terms that contribute to lo in a way that allows us to get the carry - const uint64_t r0 = loWord(plolo); - const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); - *lo = r0 + (r1 << 32); - // Sum terms contributing to hi with the carry from lo - *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; + // Each of the component 32x32 -> 64 products + const uint64_t plolo = loWord(a) * loWord(b); + const uint64_t plohi = loWord(a) * hiWord(b); + const uint64_t philo = hiWord(a) * loWord(b); + const uint64_t phihi = hiWord(a) * hiWord(b); + // Sum terms that contribute to lo in a way that allows us to get the carry + const uint64_t r0 = loWord(plolo); + const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); + *lo = r0 + (r1 << 32); + // Sum terms contributing to hi with the carry from lo + *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; } #undef loWord #undef hiWord @@ -114,32 +111,34 @@ typedef long double fp_t; #define significandBits 112 static __inline int rep_clz(rep_t a) { - const union - { - __uint128_t ll; + const union { + __uint128_t ll; #if _YUGA_BIG_ENDIAN - struct { uint64_t high, low; } s; + struct { + uint64_t high, low; + } s; #else - struct { uint64_t low, high; } s; + struct { + uint64_t low, high; + } s; #endif - } uu = { .ll = a }; + } uu = {.ll = a}; - uint64_t word; - uint64_t add; + uint64_t word; + uint64_t add; - if (uu.s.high){ - word = uu.s.high; - add = 0; - } - else{ - word = uu.s.low; - add = 64; - } - return __builtin_clzll(word) + add; + if (uu.s.high) { + word = uu.s.high; + add = 0; + } else { + word = uu.s.low; + add = 64; + } + return __builtin_clzll(word) + add; } -#define Word_LoMask UINT64_C(0x00000000ffffffff) -#define Word_HiMask UINT64_C(0xffffffff00000000) +#define Word_LoMask UINT64_C(0x00000000ffffffff) +#define Word_HiMask UINT64_C(0xffffffff00000000) #define Word_FullMask UINT64_C(0xffffffffffffffff) #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask) #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask) @@ -151,55 +150,41 @@ static __inline int rep_clz(rep_t a) { // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product11 = Word_1(a) * Word_1(b); - const uint64_t product12 = Word_1(a) * Word_2(b); - const uint64_t product13 = Word_1(a) * Word_3(b); - const uint64_t product14 = Word_1(a) * Word_4(b); - const uint64_t product21 = Word_2(a) * Word_1(b); - const uint64_t product22 = Word_2(a) * Word_2(b); - const uint64_t product23 = Word_2(a) * Word_3(b); - const uint64_t product24 = Word_2(a) * Word_4(b); - const uint64_t product31 = Word_3(a) * Word_1(b); - const uint64_t product32 = Word_3(a) * Word_2(b); - const uint64_t product33 = Word_3(a) * Word_3(b); - const uint64_t product34 = Word_3(a) * Word_4(b); - const uint64_t product41 = Word_4(a) * Word_1(b); - const uint64_t product42 = Word_4(a) * Word_2(b); - const uint64_t product43 = Word_4(a) * Word_3(b); - const uint64_t product44 = Word_4(a) * Word_4(b); - - const __uint128_t sum0 = (__uint128_t)product44; - const __uint128_t sum1 = (__uint128_t)product34 + - (__uint128_t)product43; - const __uint128_t sum2 = (__uint128_t)product24 + - (__uint128_t)product33 + - (__uint128_t)product42; - const __uint128_t sum3 = (__uint128_t)product14 + - (__uint128_t)product23 + - (__uint128_t)product32 + - (__uint128_t)product41; - const __uint128_t sum4 = (__uint128_t)product13 + - (__uint128_t)product22 + - (__uint128_t)product31; - const __uint128_t sum5 = (__uint128_t)product12 + - (__uint128_t)product21; - const __uint128_t sum6 = (__uint128_t)product11; - - const __uint128_t r0 = (sum0 & Word_FullMask) + - ((sum1 & Word_LoMask) << 32); - const __uint128_t r1 = (sum0 >> 64) + - ((sum1 >> 32) & Word_FullMask) + - (sum2 & Word_FullMask) + - ((sum3 << 32) & Word_HiMask); - - *lo = r0 + (r1 << 64); - *hi = (r1 >> 64) + - (sum1 >> 96) + - (sum2 >> 64) + - (sum3 >> 32) + - sum4 + - (sum5 << 32) + - (sum6 << 64); + const uint64_t product11 = Word_1(a) * Word_1(b); + const uint64_t product12 = Word_1(a) * Word_2(b); + const uint64_t product13 = Word_1(a) * Word_3(b); + const uint64_t product14 = Word_1(a) * Word_4(b); + const uint64_t product21 = Word_2(a) * Word_1(b); + const uint64_t product22 = Word_2(a) * Word_2(b); + const uint64_t product23 = Word_2(a) * Word_3(b); + const uint64_t product24 = Word_2(a) * Word_4(b); + const uint64_t product31 = Word_3(a) * Word_1(b); + const uint64_t product32 = Word_3(a) * Word_2(b); + const uint64_t product33 = Word_3(a) * Word_3(b); + const uint64_t product34 = Word_3(a) * Word_4(b); + const uint64_t product41 = Word_4(a) * Word_1(b); + const uint64_t product42 = Word_4(a) * Word_2(b); + const uint64_t product43 = Word_4(a) * Word_3(b); + const uint64_t product44 = Word_4(a) * Word_4(b); + + const __uint128_t sum0 = (__uint128_t)product44; + const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43; + const __uint128_t sum2 = + (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42; + const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 + + (__uint128_t)product32 + (__uint128_t)product41; + const __uint128_t sum4 = + (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31; + const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21; + const __uint128_t sum6 = (__uint128_t)product11; + + const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32); + const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) + + (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask); + + *lo = r0 + (r1 << 64); + *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 + + (sum5 << 32) + (sum6 << 64); } #undef Word_1 #undef Word_2 @@ -213,58 +198,65 @@ static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined. #endif -#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || defined(CRT_LDBL_128BIT) -#define typeWidth (sizeof(rep_t)*CHAR_BIT) -#define exponentBits (typeWidth - significandBits - 1) -#define maxExponent ((1 << exponentBits) - 1) -#define exponentBias (maxExponent >> 1) +#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \ + defined(CRT_LDBL_128BIT) +#define typeWidth (sizeof(rep_t) * CHAR_BIT) +#define exponentBits (typeWidth - significandBits - 1) +#define maxExponent ((1 << exponentBits) - 1) +#define exponentBias (maxExponent >> 1) -#define implicitBit (REP_C(1) << significandBits) +#define implicitBit (REP_C(1) << significandBits) #define significandMask (implicitBit - 1U) -#define signBit (REP_C(1) << (significandBits + exponentBits)) -#define absMask (signBit - 1U) -#define exponentMask (absMask ^ significandMask) -#define oneRep ((rep_t)exponentBias << significandBits) -#define infRep exponentMask -#define quietBit (implicitBit >> 1) -#define qnanRep (exponentMask | quietBit) +#define signBit (REP_C(1) << (significandBits + exponentBits)) +#define absMask (signBit - 1U) +#define exponentMask (absMask ^ significandMask) +#define oneRep ((rep_t)exponentBias << significandBits) +#define infRep exponentMask +#define quietBit (implicitBit >> 1) +#define qnanRep (exponentMask | quietBit) static __inline rep_t toRep(fp_t x) { - const union { fp_t f; rep_t i; } rep = {.f = x}; - return rep.i; + const union { + fp_t f; + rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline fp_t fromRep(rep_t x) { - const union { fp_t f; rep_t i; } rep = {.i = x}; - return rep.f; + const union { + fp_t f; + rep_t i; + } rep = {.i = x}; + return rep.f; } static __inline int normalize(rep_t *significand) { - const int shift = rep_clz(*significand) - rep_clz(implicitBit); - *significand <<= shift; - return 1 - shift; + const int shift = rep_clz(*significand) - rep_clz(implicitBit); + *significand <<= shift; + return 1 - shift; } static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { - *hi = *hi << count | *lo >> (typeWidth - count); - *lo = *lo << count; + *hi = *hi << count | *lo >> (typeWidth - count); + *lo = *lo << count; } -static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int count) { - if (count < typeWidth) { - const bool sticky = *lo << (typeWidth - count); - *lo = *hi << (typeWidth - count) | *lo >> count | sticky; - *hi = *hi >> count; - } - else if (count < 2*typeWidth) { - const bool sticky = *hi << (2*typeWidth - count) | *lo; - *lo = *hi >> (count - typeWidth) | sticky; - *hi = 0; - } else { - const bool sticky = *hi | *lo; - *lo = sticky; - *hi = 0; - } +static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, + unsigned int count) { + if (count < typeWidth) { + const bool sticky = *lo << (typeWidth - count); + *lo = *hi << (typeWidth - count) | *lo >> count | sticky; + *hi = *hi >> count; + } else if (count < 2 * typeWidth) { + const bool sticky = *hi << (2 * typeWidth - count) | *lo; + *lo = *hi >> (count - typeWidth) | sticky; + *hi = 0; + } else { + const bool sticky = *hi | *lo; + *lo = sticky; + *hi = 0; + } } // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids @@ -280,9 +272,9 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { // 2) 0.0 returns -inf if (exp == maxExponent) { if (((rep & signBit) == 0) || (x != x)) { - return x; // NaN or +inf: return x + return x; // NaN or +inf: return x } else { - return -x; // -inf: return -x + return -x; // -inf: return -x } } else if (x == 0.0) { // 0.0: return -inf @@ -291,13 +283,13 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { if (exp != 0) { // Normal number - return exp - exponentBias; // Unbias exponent + return exp - exponentBias; // Unbias exponent } else { // Subnormal number; normalize and repeat rep &= absMask; const int shift = 1 - normalize(&rep); exp = (rep & exponentMask) >> significandBits; - return exp - exponentBias - shift; // Unbias exponent + return exp - exponentBias - shift; // Unbias exponent } } #endif @@ -311,17 +303,17 @@ static __inline fp_t __compiler_rt_logb(fp_t x) { return __compiler_rt_logbX(x); } #elif defined(QUAD_PRECISION) - #if defined(CRT_LDBL_128BIT) +#if defined(CRT_LDBL_128BIT) static __inline fp_t __compiler_rt_logbl(fp_t x) { return __compiler_rt_logbX(x); } - #else +#else // The generic implementation only works for ieee754 floating point. For other // floating point types, continue to rely on the libm implementation for now. static __inline long double __compiler_rt_logbl(long double x) { return crt_logbl(x); } - #endif +#endif #endif #endif // FP_LIB_HEADER diff --git a/lib/builtins/fp_mul_impl.inc b/lib/builtins/fp_mul_impl.inc index b34aa1b8f..ebfc40b86 100644 --- a/lib/builtins/fp_mul_impl.inc +++ b/lib/builtins/fp_mul_impl.inc @@ -1,9 +1,8 @@ //===---- lib/fp_mul_impl.inc - floating point multiplication -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,102 +14,118 @@ #include "fp_lib.h" static __inline fp_t __mulXf3__(fp_t a, fp_t b) { - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t productSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN * anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything * NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity * non-zero = +/- infinity - if (bAbs) return fromRep(aAbs | productSign); - // infinity * zero = NaN - else return fromRep(qnanRep); - } - - if (bAbs == infRep) { - //? non-zero * infinity = +/- infinity - if (aAbs) return fromRep(bAbs | productSign); - // zero * infinity = NaN - else return fromRep(qnanRep); - } - - // zero * anything = +/- zero - if (!aAbs) return fromRep(productSign); - // anything * zero = +/- zero - if (!bAbs) return fromRep(productSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale += normalize(&bSignificand); + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t productSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN * anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything * NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity * non-zero = +/- infinity + if (bAbs) + return fromRep(aAbs | productSign); + // infinity * zero = NaN + else + return fromRep(qnanRep); } - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - - // Get the significand of a*b. Before multiplying the significands, shift - // one of them left to left-align it in the field. Thus, the product will - // have (exponentBits + 2) integral digits, all but two of which must be - // zero. Normalizing this result is just a conditional left-shift by one - // and bumping the exponent accordingly. - rep_t productHi, productLo; - wideMultiply(aSignificand, bSignificand << exponentBits, - &productHi, &productLo); - - int productExponent = aExponent + bExponent - exponentBias + scale; - - // Normalize the significand, adjust exponent if needed. - if (productHi & implicitBit) productExponent++; - else wideLeftShift(&productHi, &productLo, 1); - - // If we have overflowed the type, return +/- infinity. - if (productExponent >= maxExponent) return fromRep(infRep | productSign); - - if (productExponent <= 0) { - // Result is denormal before rounding - // - // If the result is so small that it just underflows to zero, return - // a zero of the appropriate sign. Mathematically there is no need to - // handle this case separately, but we make it a special case to - // simplify the shift logic. - const unsigned int shift = REP_C(1) - (unsigned int)productExponent; - if (shift >= typeWidth) return fromRep(productSign); - - // Otherwise, shift the significand of the result so that the round - // bit is the high bit of productLo. - wideRightShiftWithSticky(&productHi, &productLo, shift); - } - else { - // Result is normal before rounding; insert the exponent. - productHi &= significandMask; - productHi |= (rep_t)productExponent << significandBits; + if (bAbs == infRep) { + //? non-zero * infinity = +/- infinity + if (aAbs) + return fromRep(bAbs | productSign); + // zero * infinity = NaN + else + return fromRep(qnanRep); } - // Insert the sign of the result: - productHi |= productSign; - - // Final rounding. The final result may overflow to infinity, or underflow - // to zero, but those are the correct results in those cases. We use the - // default IEEE-754 round-to-nearest, ties-to-even rounding mode. - if (productLo > signBit) productHi++; - if (productLo == signBit) productHi += productHi & 1; - return fromRep(productHi); + // zero * anything = +/- zero + if (!aAbs) + return fromRep(productSign); + // anything * zero = +/- zero + if (!bAbs) + return fromRep(productSign); + + // one or both of a or b is denormal, the other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale += normalize(&bSignificand); + } + + // Or in the implicit significand bit. (If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything.) + aSignificand |= implicitBit; + bSignificand |= implicitBit; + + // Get the significand of a*b. Before multiplying the significands, shift + // one of them left to left-align it in the field. Thus, the product will + // have (exponentBits + 2) integral digits, all but two of which must be + // zero. Normalizing this result is just a conditional left-shift by one + // and bumping the exponent accordingly. + rep_t productHi, productLo; + wideMultiply(aSignificand, bSignificand << exponentBits, &productHi, + &productLo); + + int productExponent = aExponent + bExponent - exponentBias + scale; + + // Normalize the significand, adjust exponent if needed. + if (productHi & implicitBit) + productExponent++; + else + wideLeftShift(&productHi, &productLo, 1); + + // If we have overflowed the type, return +/- infinity. + if (productExponent >= maxExponent) + return fromRep(infRep | productSign); + + if (productExponent <= 0) { + // Result is denormal before rounding + // + // If the result is so small that it just underflows to zero, return + // a zero of the appropriate sign. Mathematically there is no need to + // handle this case separately, but we make it a special case to + // simplify the shift logic. + const unsigned int shift = REP_C(1) - (unsigned int)productExponent; + if (shift >= typeWidth) + return fromRep(productSign); + + // Otherwise, shift the significand of the result so that the round + // bit is the high bit of productLo. + wideRightShiftWithSticky(&productHi, &productLo, shift); + } else { + // Result is normal before rounding; insert the exponent. + productHi &= significandMask; + productHi |= (rep_t)productExponent << significandBits; + } + + // Insert the sign of the result: + productHi |= productSign; + + // Final rounding. The final result may overflow to infinity, or underflow + // to zero, but those are the correct results in those cases. We use the + // default IEEE-754 round-to-nearest, ties-to-even rounding mode. + if (productLo > signBit) + productHi++; + if (productLo == signBit) + productHi += productHi & 1; + return fromRep(productHi); } diff --git a/lib/builtins/fp_trunc.h b/lib/builtins/fp_trunc.h index d5e79bb5b..aca4c9b6e 100644 --- a/lib/builtins/fp_trunc.h +++ b/lib/builtins/fp_trunc.h @@ -1,9 +1,8 @@ //=== lib/fp_trunc.h - high precision -> low precision conversion *- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,7 +35,7 @@ static const int srcSigBits = 112; #else #error Source should be double precision or quad precision! -#endif //end source precision +#endif // end source precision #if defined DST_DOUBLE typedef double dst_t; @@ -58,19 +57,25 @@ static const int dstSigBits = 10; #else #error Destination should be single precision or double precision! -#endif //end destination precision +#endif // end destination precision // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. static __inline src_rep_t srcToRep(src_t x) { - const union { src_t f; src_rep_t i; } rep = {.f = x}; - return rep.i; + const union { + src_t f; + src_rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline dst_t dstFromRep(dst_rep_t x) { - const union { dst_t f; dst_rep_t i; } rep = {.i = x}; - return rep.f; + const union { + dst_t f; + dst_rep_t i; + } rep = {.i = x}; + return rep.f; } #endif // FP_TRUNC_HEADER diff --git a/lib/builtins/fp_trunc_impl.inc b/lib/builtins/fp_trunc_impl.inc index d88ae0609..4f103da4d 100644 --- a/lib/builtins/fp_trunc_impl.inc +++ b/lib/builtins/fp_trunc_impl.inc @@ -1,9 +1,8 @@ //= lib/fp_trunc_impl.inc - high precision -> low precision conversion *-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,96 +39,94 @@ #include "fp_trunc.h" static __inline dst_t __truncXfYf2__(src_t a) { - // Various constants whose values follow from the type parameters. - // Any reasonable optimizer will fold and propagate all of these. - const int srcBits = sizeof(src_t)*CHAR_BIT; - const int srcExpBits = srcBits - srcSigBits - 1; - const int srcInfExp = (1 << srcExpBits) - 1; - const int srcExpBias = srcInfExp >> 1; + // Various constants whose values follow from the type parameters. + // Any reasonable optimizer will fold and propagate all of these. + const int srcBits = sizeof(src_t) * CHAR_BIT; + const int srcExpBits = srcBits - srcSigBits - 1; + const int srcInfExp = (1 << srcExpBits) - 1; + const int srcExpBias = srcInfExp >> 1; - const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; - const src_rep_t srcSignificandMask = srcMinNormal - 1; - const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; - const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); - const src_rep_t srcAbsMask = srcSignMask - 1; - const src_rep_t roundMask = (SRC_REP_C(1) << (srcSigBits - dstSigBits)) - 1; - const src_rep_t halfway = SRC_REP_C(1) << (srcSigBits - dstSigBits - 1); - const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); - const src_rep_t srcNaNCode = srcQNaN - 1; + const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; + const src_rep_t srcSignificandMask = srcMinNormal - 1; + const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; + const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); + const src_rep_t srcAbsMask = srcSignMask - 1; + const src_rep_t roundMask = (SRC_REP_C(1) << (srcSigBits - dstSigBits)) - 1; + const src_rep_t halfway = SRC_REP_C(1) << (srcSigBits - dstSigBits - 1); + const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); + const src_rep_t srcNaNCode = srcQNaN - 1; - const int dstBits = sizeof(dst_t)*CHAR_BIT; - const int dstExpBits = dstBits - dstSigBits - 1; - const int dstInfExp = (1 << dstExpBits) - 1; - const int dstExpBias = dstInfExp >> 1; + const int dstBits = sizeof(dst_t) * CHAR_BIT; + const int dstExpBits = dstBits - dstSigBits - 1; + const int dstInfExp = (1 << dstExpBits) - 1; + const int dstExpBias = dstInfExp >> 1; - const int underflowExponent = srcExpBias + 1 - dstExpBias; - const int overflowExponent = srcExpBias + dstInfExp - dstExpBias; - const src_rep_t underflow = (src_rep_t)underflowExponent << srcSigBits; - const src_rep_t overflow = (src_rep_t)overflowExponent << srcSigBits; + const int underflowExponent = srcExpBias + 1 - dstExpBias; + const int overflowExponent = srcExpBias + dstInfExp - dstExpBias; + const src_rep_t underflow = (src_rep_t)underflowExponent << srcSigBits; + const src_rep_t overflow = (src_rep_t)overflowExponent << srcSigBits; - const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigBits - 1); - const dst_rep_t dstNaNCode = dstQNaN - 1; + const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigBits - 1); + const dst_rep_t dstNaNCode = dstQNaN - 1; - // Break a into a sign and representation of the absolute value - const src_rep_t aRep = srcToRep(a); - const src_rep_t aAbs = aRep & srcAbsMask; - const src_rep_t sign = aRep & srcSignMask; - dst_rep_t absResult; + // Break a into a sign and representation of the absolute value + const src_rep_t aRep = srcToRep(a); + const src_rep_t aAbs = aRep & srcAbsMask; + const src_rep_t sign = aRep & srcSignMask; + dst_rep_t absResult; - if (aAbs - underflow < aAbs - overflow) { - // The exponent of a is within the range of normal numbers in the - // destination format. We can convert by simply right-shifting with - // rounding and adjusting the exponent. - absResult = aAbs >> (srcSigBits - dstSigBits); - absResult -= (dst_rep_t)(srcExpBias - dstExpBias) << dstSigBits; + if (aAbs - underflow < aAbs - overflow) { + // The exponent of a is within the range of normal numbers in the + // destination format. We can convert by simply right-shifting with + // rounding and adjusting the exponent. + absResult = aAbs >> (srcSigBits - dstSigBits); + absResult -= (dst_rep_t)(srcExpBias - dstExpBias) << dstSigBits; - const src_rep_t roundBits = aAbs & roundMask; - // Round to nearest - if (roundBits > halfway) - absResult++; - // Ties to even - else if (roundBits == halfway) - absResult += absResult & 1; - } - else if (aAbs > srcInfinity) { - // a is NaN. - // Conjure the result by beginning with infinity, setting the qNaN - // bit and inserting the (truncated) trailing NaN field. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - absResult |= dstQNaN; - absResult |= ((aAbs & srcNaNCode) >> (srcSigBits - dstSigBits)) & dstNaNCode; - } - else if (aAbs >= overflow) { - // a overflows to infinity. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - } - else { - // a underflows on conversion to the destination type or is an exact - // zero. The result may be a denormal or zero. Extract the exponent - // to get the shift amount for the denormalization. - const int aExp = aAbs >> srcSigBits; - const int shift = srcExpBias - dstExpBias - aExp + 1; + const src_rep_t roundBits = aAbs & roundMask; + // Round to nearest + if (roundBits > halfway) + absResult++; + // Ties to even + else if (roundBits == halfway) + absResult += absResult & 1; + } else if (aAbs > srcInfinity) { + // a is NaN. + // Conjure the result by beginning with infinity, setting the qNaN + // bit and inserting the (truncated) trailing NaN field. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + absResult |= dstQNaN; + absResult |= + ((aAbs & srcNaNCode) >> (srcSigBits - dstSigBits)) & dstNaNCode; + } else if (aAbs >= overflow) { + // a overflows to infinity. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + } else { + // a underflows on conversion to the destination type or is an exact + // zero. The result may be a denormal or zero. Extract the exponent + // to get the shift amount for the denormalization. + const int aExp = aAbs >> srcSigBits; + const int shift = srcExpBias - dstExpBias - aExp + 1; - const src_rep_t significand = (aRep & srcSignificandMask) | srcMinNormal; + const src_rep_t significand = (aRep & srcSignificandMask) | srcMinNormal; - // Right shift by the denormalization amount with sticky. - if (shift > srcSigBits) { - absResult = 0; - } else { - const bool sticky = significand << (srcBits - shift); - src_rep_t denormalizedSignificand = significand >> shift | sticky; - absResult = denormalizedSignificand >> (srcSigBits - dstSigBits); - const src_rep_t roundBits = denormalizedSignificand & roundMask; - // Round to nearest - if (roundBits > halfway) - absResult++; - // Ties to even - else if (roundBits == halfway) - absResult += absResult & 1; - } + // Right shift by the denormalization amount with sticky. + if (shift > srcSigBits) { + absResult = 0; + } else { + const bool sticky = significand << (srcBits - shift); + src_rep_t denormalizedSignificand = significand >> shift | sticky; + absResult = denormalizedSignificand >> (srcSigBits - dstSigBits); + const src_rep_t roundBits = denormalizedSignificand & roundMask; + // Round to nearest + if (roundBits > halfway) + absResult++; + // Ties to even + else if (roundBits == halfway) + absResult += absResult & 1; } + } - // Apply the signbit to (dst_t)abs(a). - const dst_rep_t result = absResult | sign >> (srcBits - dstBits); - return dstFromRep(result); + // Apply the signbit to (dst_t)abs(a). + const dst_rep_t result = absResult | sign >> (srcBits - dstBits); + return dstFromRep(result); } diff --git a/lib/builtins/gcc_personality_v0.c b/lib/builtins/gcc_personality_v0.c index 68581ef16..d12ee03c4 100644 --- a/lib/builtins/gcc_personality_v0.c +++ b/lib/builtins/gcc_personality_v0.c @@ -1,145 +1,135 @@ -/* ===-- gcc_personality_v0.c - Implement __gcc_personality_v0 -------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - */ +//===-- gcc_personality_v0.c - Implement __gcc_personality_v0 -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <unwind.h> -#if defined(__arm__) && !defined(__ARM_DWARF_EH__) && !defined(__USING_SJLJ_EXCEPTIONS__) -/* - * When building with older compilers (e.g. clang <3.9), it is possible that we - * have a version of unwind.h which does not provide the EHABI declarations - * which are quired for the C personality to conform to the specification. In - * order to provide forward compatibility for such compilers, we re-declare the - * necessary interfaces in the helper to permit a standalone compilation of the - * builtins (which contains the C unwinding personality for historical reasons). - */ +#if defined(__arm__) && !defined(__ARM_DWARF_EH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__) +// When building with older compilers (e.g. clang <3.9), it is possible that we +// have a version of unwind.h which does not provide the EHABI declarations +// which are quired for the C personality to conform to the specification. In +// order to provide forward compatibility for such compilers, we re-declare the +// necessary interfaces in the helper to permit a standalone compilation of the +// builtins (which contains the C unwinding personality for historical reasons). #include "unwind-ehabi-helpers.h" #endif -/* - * Pointer encodings documented at: - * http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html - */ - -#define DW_EH_PE_omit 0xff /* no data follows */ - -#define DW_EH_PE_absptr 0x00 -#define DW_EH_PE_uleb128 0x01 -#define DW_EH_PE_udata2 0x02 -#define DW_EH_PE_udata4 0x03 -#define DW_EH_PE_udata8 0x04 -#define DW_EH_PE_sleb128 0x09 -#define DW_EH_PE_sdata2 0x0A -#define DW_EH_PE_sdata4 0x0B -#define DW_EH_PE_sdata8 0x0C - -#define DW_EH_PE_pcrel 0x10 -#define DW_EH_PE_textrel 0x20 -#define DW_EH_PE_datarel 0x30 -#define DW_EH_PE_funcrel 0x40 -#define DW_EH_PE_aligned 0x50 -#define DW_EH_PE_indirect 0x80 /* gcc extension */ - - - -/* read a uleb128 encoded value and advance pointer */ -static uintptr_t readULEB128(const uint8_t** data) -{ - uintptr_t result = 0; - uintptr_t shift = 0; - unsigned char byte; - const uint8_t* p = *data; - do { - byte = *p++; - result |= (byte & 0x7f) << shift; - shift += 7; - } while (byte & 0x80); - *data = p; - return result; +// Pointer encodings documented at: +// http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html + +#define DW_EH_PE_omit 0xff // no data follows + +#define DW_EH_PE_absptr 0x00 +#define DW_EH_PE_uleb128 0x01 +#define DW_EH_PE_udata2 0x02 +#define DW_EH_PE_udata4 0x03 +#define DW_EH_PE_udata8 0x04 +#define DW_EH_PE_sleb128 0x09 +#define DW_EH_PE_sdata2 0x0A +#define DW_EH_PE_sdata4 0x0B +#define DW_EH_PE_sdata8 0x0C + +#define DW_EH_PE_pcrel 0x10 +#define DW_EH_PE_textrel 0x20 +#define DW_EH_PE_datarel 0x30 +#define DW_EH_PE_funcrel 0x40 +#define DW_EH_PE_aligned 0x50 +#define DW_EH_PE_indirect 0x80 // gcc extension + +// read a uleb128 encoded value and advance pointer +static uintptr_t readULEB128(const uint8_t **data) { + uintptr_t result = 0; + uintptr_t shift = 0; + unsigned char byte; + const uint8_t *p = *data; + do { + byte = *p++; + result |= (byte & 0x7f) << shift; + shift += 7; + } while (byte & 0x80); + *data = p; + return result; } -/* read a pointer encoded value and advance pointer */ -static uintptr_t readEncodedPointer(const uint8_t** data, uint8_t encoding) -{ - const uint8_t* p = *data; - uintptr_t result = 0; - - if ( encoding == DW_EH_PE_omit ) - return 0; - - /* first get value */ - switch (encoding & 0x0F) { - case DW_EH_PE_absptr: - result = *((const uintptr_t*)p); - p += sizeof(uintptr_t); - break; - case DW_EH_PE_uleb128: - result = readULEB128(&p); - break; - case DW_EH_PE_udata2: - result = *((const uint16_t*)p); - p += sizeof(uint16_t); - break; - case DW_EH_PE_udata4: - result = *((const uint32_t*)p); - p += sizeof(uint32_t); - break; - case DW_EH_PE_udata8: - result = *((const uint64_t*)p); - p += sizeof(uint64_t); - break; - case DW_EH_PE_sdata2: - result = *((const int16_t*)p); - p += sizeof(int16_t); - break; - case DW_EH_PE_sdata4: - result = *((const int32_t*)p); - p += sizeof(int32_t); - break; - case DW_EH_PE_sdata8: - result = *((const int64_t*)p); - p += sizeof(int64_t); - break; - case DW_EH_PE_sleb128: - default: - /* not supported */ - compilerrt_abort(); - break; - } - - /* then add relative offset */ - switch ( encoding & 0x70 ) { - case DW_EH_PE_absptr: - /* do nothing */ - break; - case DW_EH_PE_pcrel: - result += (uintptr_t)(*data); - break; - case DW_EH_PE_textrel: - case DW_EH_PE_datarel: - case DW_EH_PE_funcrel: - case DW_EH_PE_aligned: - default: - /* not supported */ - compilerrt_abort(); - break; - } - - /* then apply indirection */ - if (encoding & DW_EH_PE_indirect) { - result = *((const uintptr_t*)result); - } - - *data = p; - return result; +// read a pointer encoded value and advance pointer +static uintptr_t readEncodedPointer(const uint8_t **data, uint8_t encoding) { + const uint8_t *p = *data; + uintptr_t result = 0; + + if (encoding == DW_EH_PE_omit) + return 0; + + // first get value + switch (encoding & 0x0F) { + case DW_EH_PE_absptr: + result = *((const uintptr_t *)p); + p += sizeof(uintptr_t); + break; + case DW_EH_PE_uleb128: + result = readULEB128(&p); + break; + case DW_EH_PE_udata2: + result = *((const uint16_t *)p); + p += sizeof(uint16_t); + break; + case DW_EH_PE_udata4: + result = *((const uint32_t *)p); + p += sizeof(uint32_t); + break; + case DW_EH_PE_udata8: + result = *((const uint64_t *)p); + p += sizeof(uint64_t); + break; + case DW_EH_PE_sdata2: + result = *((const int16_t *)p); + p += sizeof(int16_t); + break; + case DW_EH_PE_sdata4: + result = *((const int32_t *)p); + p += sizeof(int32_t); + break; + case DW_EH_PE_sdata8: + result = *((const int64_t *)p); + p += sizeof(int64_t); + break; + case DW_EH_PE_sleb128: + default: + // not supported + compilerrt_abort(); + break; + } + + // then add relative offset + switch (encoding & 0x70) { + case DW_EH_PE_absptr: + // do nothing + break; + case DW_EH_PE_pcrel: + result += (uintptr_t)(*data); + break; + case DW_EH_PE_textrel: + case DW_EH_PE_datarel: + case DW_EH_PE_funcrel: + case DW_EH_PE_aligned: + default: + // not supported + compilerrt_abort(); + break; + } + + // then apply indirection + if (encoding & DW_EH_PE_indirect) { + result = *((const uintptr_t *)result); + } + + *data = p; + return result; } #if defined(__arm__) && !defined(__USING_SJLJ_EXCEPTIONS__) && \ @@ -153,99 +143,92 @@ static inline _Unwind_Reason_Code continueUnwind(struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) { #if USING_ARM_EHABI - /* - * On ARM EHABI the personality routine is responsible for actually - * unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). - */ - if (__gnu_unwind_frame(exceptionObject, context) != _URC_OK) - return _URC_FAILURE; + // On ARM EHABI the personality routine is responsible for actually + // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). + if (__gnu_unwind_frame(exceptionObject, context) != _URC_OK) + return _URC_FAILURE; #endif - return _URC_CONTINUE_UNWIND; + return _URC_CONTINUE_UNWIND; } -/* - * The C compiler makes references to __gcc_personality_v0 in - * the dwarf unwind information for translation units that use - * __attribute__((cleanup(xx))) on local variables. - * This personality routine is called by the system unwinder - * on each frame as the stack is unwound during a C++ exception - * throw through a C function compiled with -fexceptions. - */ +// The C compiler makes references to __gcc_personality_v0 in +// the dwarf unwind information for translation units that use +// __attribute__((cleanup(xx))) on local variables. +// This personality routine is called by the system unwinder +// on each frame as the stack is unwound during a C++ exception +// throw through a C function compiled with -fexceptions. #if __USING_SJLJ_EXCEPTIONS__ -/* the setjump-longjump based exceptions personality routine has a - * different name */ -COMPILER_RT_ABI _Unwind_Reason_Code -__gcc_personality_sj0(int version, _Unwind_Action actions, - uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - struct _Unwind_Context *context) +// the setjump-longjump based exceptions personality routine has a +// different name +COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_sj0( + int version, _Unwind_Action actions, uint64_t exceptionClass, + struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) #elif USING_ARM_EHABI -/* The ARM EHABI personality routine has a different signature. */ +// The ARM EHABI personality routine has a different signature. COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( - _Unwind_State state, struct _Unwind_Exception *exceptionObject, - struct _Unwind_Context *context) + _Unwind_State state, struct _Unwind_Exception *exceptionObject, + struct _Unwind_Context *context) #else -COMPILER_RT_ABI _Unwind_Reason_Code -__gcc_personality_v0(int version, _Unwind_Action actions, - uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - struct _Unwind_Context *context) +COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( + int version, _Unwind_Action actions, uint64_t exceptionClass, + struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) #endif { - /* Since C does not have catch clauses, there is nothing to do during */ - /* phase 1 (the search phase). */ + // Since C does not have catch clauses, there is nothing to do during + // phase 1 (the search phase). #if USING_ARM_EHABI - /* After resuming from a cleanup we should also continue on to the next - * frame straight away. */ - if ((state & _US_ACTION_MASK) != _US_UNWIND_FRAME_STARTING) + // After resuming from a cleanup we should also continue on to the next + // frame straight away. + if ((state & _US_ACTION_MASK) != _US_UNWIND_FRAME_STARTING) #else - if ( actions & _UA_SEARCH_PHASE ) + if (actions & _UA_SEARCH_PHASE) #endif - return continueUnwind(exceptionObject, context); - - /* There is nothing to do if there is no LSDA for this frame. */ - const uint8_t* lsda = (uint8_t*)_Unwind_GetLanguageSpecificData(context); - if ( lsda == (uint8_t*) 0 ) - return continueUnwind(exceptionObject, context); + return continueUnwind(exceptionObject, context); - uintptr_t pc = (uintptr_t)_Unwind_GetIP(context)-1; - uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); - uintptr_t pcOffset = pc - funcStart; + // There is nothing to do if there is no LSDA for this frame. + const uint8_t *lsda = (uint8_t *)_Unwind_GetLanguageSpecificData(context); + if (lsda == (uint8_t *)0) + return continueUnwind(exceptionObject, context); - /* Parse LSDA header. */ - uint8_t lpStartEncoding = *lsda++; - if (lpStartEncoding != DW_EH_PE_omit) { - readEncodedPointer(&lsda, lpStartEncoding); - } - uint8_t ttypeEncoding = *lsda++; - if (ttypeEncoding != DW_EH_PE_omit) { - readULEB128(&lsda); - } - /* Walk call-site table looking for range that includes current PC. */ - uint8_t callSiteEncoding = *lsda++; - uint32_t callSiteTableLength = readULEB128(&lsda); - const uint8_t* callSiteTableStart = lsda; - const uint8_t* callSiteTableEnd = callSiteTableStart + callSiteTableLength; - const uint8_t* p=callSiteTableStart; - while (p < callSiteTableEnd) { - uintptr_t start = readEncodedPointer(&p, callSiteEncoding); - uintptr_t length = readEncodedPointer(&p, callSiteEncoding); - uintptr_t landingPad = readEncodedPointer(&p, callSiteEncoding); - readULEB128(&p); /* action value not used for C code */ - if ( landingPad == 0 ) - continue; /* no landing pad for this entry */ - if ( (start <= pcOffset) && (pcOffset < (start+length)) ) { - /* Found landing pad for the PC. - * Set Instruction Pointer to so we re-enter function - * at landing pad. The landing pad is created by the compiler - * to take two parameters in registers. - */ - _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), - (uintptr_t)exceptionObject); - _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); - _Unwind_SetIP(context, (funcStart + landingPad)); - return _URC_INSTALL_CONTEXT; - } + uintptr_t pc = (uintptr_t)_Unwind_GetIP(context) - 1; + uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); + uintptr_t pcOffset = pc - funcStart; + + // Parse LSDA header. + uint8_t lpStartEncoding = *lsda++; + if (lpStartEncoding != DW_EH_PE_omit) { + readEncodedPointer(&lsda, lpStartEncoding); + } + uint8_t ttypeEncoding = *lsda++; + if (ttypeEncoding != DW_EH_PE_omit) { + readULEB128(&lsda); + } + // Walk call-site table looking for range that includes current PC. + uint8_t callSiteEncoding = *lsda++; + uint32_t callSiteTableLength = readULEB128(&lsda); + const uint8_t *callSiteTableStart = lsda; + const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength; + const uint8_t *p = callSiteTableStart; + while (p < callSiteTableEnd) { + uintptr_t start = readEncodedPointer(&p, callSiteEncoding); + uintptr_t length = readEncodedPointer(&p, callSiteEncoding); + uintptr_t landingPad = readEncodedPointer(&p, callSiteEncoding); + readULEB128(&p); // action value not used for C code + if (landingPad == 0) + continue; // no landing pad for this entry + if ((start <= pcOffset) && (pcOffset < (start + length))) { + // Found landing pad for the PC. + // Set Instruction Pointer to so we re-enter function + // at landing pad. The landing pad is created by the compiler + // to take two parameters in registers. + _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), + (uintptr_t)exceptionObject); + _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); + _Unwind_SetIP(context, (funcStart + landingPad)); + return _URC_INSTALL_CONTEXT; } + } - /* No landing pad found, continue unwinding. */ - return continueUnwind(exceptionObject, context); + // No landing pad found, continue unwinding. + return continueUnwind(exceptionObject, context); } diff --git a/lib/builtins/hexagon/common_entry_exit_abi1.S b/lib/builtins/hexagon/common_entry_exit_abi1.S index d5479d2a5..23fed01c6 100644 --- a/lib/builtins/hexagon/common_entry_exit_abi1.S +++ b/lib/builtins/hexagon/common_entry_exit_abi1.S @@ -1,14 +1,13 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .text @@ -33,16 +32,16 @@ -/* Save r25:24 at fp+#-8 and r27:26 at fp+#-16. */ +// Save r25:24 at fp+#-8 and r27:26 at fp+#-16. -/* The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. -/* Since we can only issue one store per packet, we don't hurt performance by - simply jumping to the right point in this sequence of stores. */ +// Since we can only issue one store per packet, we don't hurt performance by +// simply jumping to the right point in this sequence of stores. FUNCTION_BEGIN __save_r24_through_r27 memd(fp+#-16) = r27:26 @@ -56,10 +55,10 @@ FUNCTION_END __save_r24_through_r25 -/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the tail call. */ +// For each of the *_before_tailcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the tail call. FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe_before_tailcall r27:26 = memd(fp+#-16) @@ -74,8 +73,8 @@ FUNCTION_END __restore_r24_through_r25_and_deallocframe_before_tailcall -/* Here we use the extra load bandwidth to restore LR early, allowing the return - to occur in parallel with the deallocframe. */ +// Here we use the extra load bandwidth to restore LR early, allowing the return +// to occur in parallel with the deallocframe. FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe { @@ -92,7 +91,7 @@ FUNCTION_END __restore_r24_through_r27_and_deallocframe -/* Here the load bandwidth is maximized. */ +// Here the load bandwidth is maximized. FUNCTION_BEGIN __restore_r24_through_r25_and_deallocframe { diff --git a/lib/builtins/hexagon/common_entry_exit_abi2.S b/lib/builtins/hexagon/common_entry_exit_abi2.S index 6f470343d..3b85aea2f 100644 --- a/lib/builtins/hexagon/common_entry_exit_abi2.S +++ b/lib/builtins/hexagon/common_entry_exit_abi2.S @@ -1,14 +1,13 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .p2align 2 @@ -33,10 +32,10 @@ -/* Save r17:16 at fp+#-8, r19:18 at fp+#-16, r21:20 at fp+#-24, r23:22 at - fp+#-32, r25:24 at fp+#-40, and r27:26 at fp+#-48. - The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// Save r17:16 at fp+#-8, r19:18 at fp+#-16, r21:20 at fp+#-24, r23:22 at +// fp+#-32, r25:24 at fp+#-40, and r27:26 at fp+#-48. +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. FUNCTION_BEGIN __save_r16_through_r27 { @@ -107,10 +106,10 @@ FUNCTION_BEGIN __save_r16_through_r17 } FUNCTION_END __save_r16_through_r17 -/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the tail call. */ +// For each of the *_before_tailcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the tail call. FUNCTION_BEGIN __restore_r16_through_r27_and_deallocframe_before_tailcall diff --git a/lib/builtins/hexagon/common_entry_exit_legacy.S b/lib/builtins/hexagon/common_entry_exit_legacy.S index 3258f15a3..8a6044573 100644 --- a/lib/builtins/hexagon/common_entry_exit_legacy.S +++ b/lib/builtins/hexagon/common_entry_exit_legacy.S @@ -1,15 +1,14 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .text @@ -34,17 +33,17 @@ -/* Save r27:26 at fp+#-8, r25:24 at fp+#-16, r23:22 at fp+#-24, r21:20 at - fp+#-32, r19:18 at fp+#-40, and r17:16 at fp+#-48. */ +// Save r27:26 at fp+#-8, r25:24 at fp+#-16, r23:22 at fp+#-24, r21:20 at +// fp+#-32, r19:18 at fp+#-40, and r17:16 at fp+#-48. -/* The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. -/* Since we can only issue one store per packet, we don't hurt performance by - simply jumping to the right point in this sequence of stores. */ +// Since we can only issue one store per packet, we don't hurt performance by +// simply jumping to the right point in this sequence of stores. FUNCTION_BEGIN __save_r27_through_r16 memd(fp+#-48) = r17:16 @@ -65,10 +64,10 @@ FUNCTION_END __save_r27_through_r24 -/* For each of the *_before_sibcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the sibcall. */ +// For each of the *_before_sibcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the sibcall. FUNCTION_BEGIN __restore_r27_through_r20_and_deallocframe_before_sibcall { @@ -108,8 +107,8 @@ FUNCTION_END __restore_r27_through_r26_and_deallocframe_before_sibcall -/* Here we use the extra load bandwidth to restore LR early, allowing the return - to occur in parallel with the deallocframe. */ +// Here we use the extra load bandwidth to restore LR early, allowing the return +// to occur in parallel with the deallocframe. FUNCTION_BEGIN __restore_r27_through_r16_and_deallocframe { @@ -136,7 +135,7 @@ FUNCTION_END __restore_r27_through_r24_and_deallocframe -/* Here the load bandwidth is maximized for all three functions. */ +// Here the load bandwidth is maximized for all three functions. FUNCTION_BEGIN __restore_r27_through_r18_and_deallocframe { diff --git a/lib/builtins/hexagon/dfaddsub.S b/lib/builtins/hexagon/dfaddsub.S index 4173f86a4..1b0d34550 100644 --- a/lib/builtins/hexagon/dfaddsub.S +++ b/lib/builtins/hexagon/dfaddsub.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 #define AH r1 @@ -179,18 +178,17 @@ __hexagon_subdf3: .Ladd_ovf_unf: // Overflow or Denormal is possible // Good news: Underflow flag is not possible! - /* - * ATMP has 2's complement value - * - * EXPA has A's exponent, EXPB has EXPA-BIAS-60 - * - * Convert, extract exponent, add adjustment. - * If > 2046, overflow - * If <= 0, denormal - * - * Note that we've not done our zero check yet, so do that too - * - */ + + // ATMP has 2's complement value + // + // EXPA has A's exponent, EXPB has EXPA-BIAS-60 + // + // Convert, extract exponent, add adjustment. + // If > 2046, overflow + // If <= 0, denormal + // + // Note that we've not done our zero check yet, so do that too + { A = convert_d2df(ATMP) p0 = cmp.eq(ATMPH,#0) diff --git a/lib/builtins/hexagon/dfdiv.S b/lib/builtins/hexagon/dfdiv.S index 0c5dbe272..202965ec4 100644 --- a/lib/builtins/hexagon/dfdiv.S +++ b/lib/builtins/hexagon/dfdiv.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Divide */ +// Double Precision Divide #define A r1:0 #define AH r1 @@ -237,10 +236,10 @@ __hexagon_divdf3: P_TMP = cmp.gt(EXPA,#0) if (P_TMP.new) jump:nt .Lpossible_unf // round up to normal possible... } - /* Underflow */ - /* We know what the infinite range exponent should be (EXPA) */ - /* Q is 2's complement, PROD is abs(Q) */ - /* Normalize Q, shift right, add a high bit, convert, change exponent */ + // Underflow + // We know what the infinite range exponent should be (EXPA) + // Q is 2's complement, PROD is abs(Q) + // Normalize Q, shift right, add a high bit, convert, change exponent #define FUDGE1 7 // how much to shift right #define FUDGE2 4 // how many guard/round to keep at lsbs @@ -287,8 +286,8 @@ __hexagon_divdf3: .Lpossible_unf: - /* If upper parts of Q were all F's, but abs(A) == 0x00100000_00000000, we rounded up to min_normal */ - /* The answer is correct, but we need to raise Underflow */ + // If upper parts of Q were all F's, but abs(A) == 0x00100000_00000000, we rounded up to min_normal + // The answer is correct, but we need to raise Underflow { B = extractu(A,#63,#0) TMPPAIR = combine(##0x00100000,#0) // min normal @@ -321,9 +320,9 @@ __hexagon_divdf3: } .Ldiv_ovf: - /* - * Raise Overflow, and choose the correct overflow value (saturated normal or infinity) - */ + + // Raise Overflow, and choose the correct overflow value (saturated normal or infinity) + { TMP = USR B = combine(##0x7fefffff,#-1) @@ -389,8 +388,8 @@ __hexagon_divdf3: if (!P_ZERO) jump .Ldiv_zero_result if (!P_INF) jump .Ldiv_inf_result } - /* Now we've narrowed it down to (de)normal / (de)normal */ - /* Set up A/EXPA B/EXPB and go back */ + // Now we've narrowed it down to (de)normal / (de)normal + // Set up A/EXPA B/EXPB and go back #undef P_ZERO #undef P_INF #define P_TMP2 p1 diff --git a/lib/builtins/hexagon/dffma.S b/lib/builtins/hexagon/dffma.S index 97b885a3b..c201d3d8b 100644 --- a/lib/builtins/hexagon/dffma.S +++ b/lib/builtins/hexagon/dffma.S @@ -1,16 +1,15 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG #define END(TAG) .size TAG,.-TAG -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 @@ -76,33 +75,29 @@ #define SR_ROUND_OFF 22 #endif - /* - * First, classify for normal values, and abort if abnormal - * - * Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8 - * - * Since we know that the 2 MSBs of the H registers is zero, we should never carry - * the partial products that involve the H registers - * - * Try to buy X slots, at the expense of latency if needed - * - * We will have PP_HH with the upper bits of the product, PP_LL with the lower - * PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x0100_0000_0000_0000 - * - * 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS - * - * We need to align CTMP. - * If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add - * If CTMP << PP align CTMP and add 128 bits. Then compute sticky - * If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation. - * - * Convert partial product and CTMP to 2's complement prior to addition - * - * After we add, we need to normalize into upper 64 bits, then compute sticky. - * - * - */ + // First, classify for normal values, and abort if abnormal + // + // Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8 + // + // Since we know that the 2 MSBs of the H registers is zero, we should never carry + // the partial products that involve the H registers + // + // Try to buy X slots, at the expense of latency if needed + // + // We will have PP_HH with the upper bits of the product, PP_LL with the lower + // PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x0100_0000_0000_0000 + // + // 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS + // + // We need to align CTMP. + // If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add + // If CTMP << PP align CTMP and add 128 bits. Then compute sticky + // If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation. + // + // Convert partial product and CTMP to 2's complement prior to addition + // + // After we add, we need to normalize into upper 64 bits, then compute sticky. .text .global __hexagon_fmadf4 @@ -182,14 +177,12 @@ fma: #define EXPCA r19:18 EXPC = extractu(CH,#EXPBITS,#HI_MANTBITS) } - /* PP_HH:PP_LL now has product */ - /* CTMP is negated */ - /* EXPA,B,C are extracted */ - /* - * We need to negate PP - * Since we will be adding with carry later, if we need to negate, - * just invert all bits now, which we can do conditionally and in parallel - */ + // PP_HH:PP_LL now has product + // CTMP is negated + // EXPA,B,C are extracted + // We need to negate PP + // Since we will be adding with carry later, if we need to negate, + // just invert all bits now, which we can do conditionally and in parallel #define PP_HH_TMP r15:14 #define PP_LL_TMP r7:6 { @@ -274,18 +267,16 @@ fma: PP_HH = add(CTMP,PP_HH,P_CARRY):carry TMP = #62 } - /* - * PP_HH:PP_LL now holds the sum - * We may need to normalize left, up to ??? bits. - * - * I think that if we have massive cancellation, the range we normalize by - * is still limited - */ + // PP_HH:PP_LL now holds the sum + // We may need to normalize left, up to ??? bits. + // + // I think that if we have massive cancellation, the range we normalize by + // is still limited { LEFTSHIFT = add(clb(PP_HH),#-2) if (!cmp.eq(LEFTSHIFT.new,TMP)) jump:t 1f // all sign bits? } - /* We had all sign bits, shift left by 62. */ + // We had all sign bits, shift left by 62. { CTMP = extractu(PP_LL,#62,#2) PP_LL = asl(PP_LL,#62) @@ -330,7 +321,7 @@ fma: if (!P_TMP) dealloc_return // not zero, return } .Ladd_yields_zero: - /* We had full cancellation. Return +/- zero (-0 when round-down) */ + // We had full cancellation. Return +/- zero (-0 when round-down) { TMP = USR A = #0 @@ -408,9 +399,9 @@ fma: EXPA = sub(#1+5,TMP) // Amount to right shift to denormalize p3 = cmp.gt(CTMPH,#-1) } - /* Underflow */ - /* We know that the infinte range exponent should be EXPA */ - /* CTMP is 2's complement, ATMP is abs(CTMP) */ + // Underflow + // We know that the infinte range exponent should be EXPA + // CTMP is 2's complement, ATMP is abs(CTMP) { EXPA = add(EXPA,EXPB) // how much to shift back right ATMP = asl(ATMP,EXPB) // shift left @@ -593,7 +584,7 @@ fma: p1 = dfclass(C,#0x08) if (p1.new) jump:nt .Lfma_inf_plus_inf } - /* A*B is +/- inf, C is finite. Return A */ + // A*B is +/- inf, C is finite. Return A { jumpr r31 } @@ -649,7 +640,7 @@ fma: if (!p0) A = C // If C is not zero, return C if (!p0) jumpr r31 } - /* B has correctly signed zero, C is also zero */ + // B has correctly signed zero, C is also zero .Lzero_plus_zero: { p0 = cmp.eq(B,C) // yes, scalar equals. +0++0 or -0+-0 @@ -674,8 +665,8 @@ fma: #define CTMP r11:10 .falign .Lfma_abnormal_c: - /* We know that AB is normal * normal */ - /* C is not normal: zero, subnormal, inf, or NaN. */ + // We know that AB is normal * normal + // C is not normal: zero, subnormal, inf, or NaN. { p0 = dfclass(C,#0x10) // is C NaN? if (p0.new) jump:nt .Lnan diff --git a/lib/builtins/hexagon/dfminmax.S b/lib/builtins/hexagon/dfminmax.S index 41122911f..44f031ba1 100644 --- a/lib/builtins/hexagon/dfminmax.S +++ b/lib/builtins/hexagon/dfminmax.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,17 +14,14 @@ #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG #define END(TAG) .size TAG,.-TAG -/* - * Min and Max return A if B is NaN, or B if A is NaN - * Otherwise, they return the smaller or bigger value - * - * If values are equal, we want to favor -0.0 for min and +0.0 for max. - */ +// Min and Max return A if B is NaN, or B if A is NaN +// Otherwise, they return the smaller or bigger value +// +// If values are equal, we want to favor -0.0 for min and +0.0 for max. + +// Compares always return false for NaN +// if (isnan(A)) A = B; if (A > B) A = B will only trigger at most one of those options. -/* - * Compares always return false for NaN - * if (isnan(A)) A = B; if (A > B) A = B will only trigger at most one of those options. - */ .text .global __hexagon_mindf3 .global __hexagon_maxdf3 @@ -51,7 +47,7 @@ fmin: p2 = dfcmp.eq(A,B) // if A == B if (!p2.new) jumpr:t r31 } - /* A == B, return A|B to select -0.0 over 0.0 */ + // A == B, return A|B to select -0.0 over 0.0 { A = or(ATMP,B) jumpr r31 @@ -71,7 +67,7 @@ fmax: p2 = dfcmp.eq(A,B) if (!p2.new) jumpr:t r31 } - /* A == B, return A&B to select 0.0 over -0.0 */ + // A == B, return A&B to select 0.0 over -0.0 { A = and(ATMP,B) jumpr r31 diff --git a/lib/builtins/hexagon/dfmul.S b/lib/builtins/hexagon/dfmul.S index fde6d77bd..e6f62c351 100644 --- a/lib/builtins/hexagon/dfmul.S +++ b/lib/builtins/hexagon/dfmul.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 #define AH r1 #define AL r0 @@ -47,8 +46,8 @@ #define BIAS 1024 #define MANTISSA_TO_INT_BIAS 52 -/* Some constant to adjust normalization amount in error code */ -/* Amount to right shift the partial product to get to a denorm */ +// Some constant to adjust normalization amount in error code +// Amount to right shift the partial product to get to a denorm #define FUDGE 5 #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG @@ -80,10 +79,10 @@ __hexagon_muldf3: PP_ODD = mpyu(BTMPL,ATMPH) BTMP = insert(ONE,#2,#62) } - /* since we know that the MSB of the H registers is zero, we should never carry */ - /* H <= 2^31-1. L <= 2^32-1. Therefore, HL <= 2^63-2^32-2^31+1 */ - /* Adding 2 HLs, we get 2^64-3*2^32+2 maximum. */ - /* Therefore, we can add 3 2^32-1 values safely without carry. We only need one. */ + // since we know that the MSB of the H registers is zero, we should never carry + // H <= 2^31-1. L <= 2^32-1. Therefore, HL <= 2^63-2^32-2^31+1 + // Adding 2 HLs, we get 2^64-3*2^32+2 maximum. + // Therefore, we can add 3 2^32-1 values safely without carry. We only need one. { PP_LL = mpyu(ATMPL,BTMPL) PP_ODD += mpyu(ATMPL,BTMPH) @@ -99,10 +98,10 @@ __hexagon_muldf3: p1 = cmp.eq(PP_LL_L,#0) // 64 lsb's 0? p1 = cmp.eq(PP_ODD_L,#0) // 64 lsb's 0? } - /* - * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so - */ + + // PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x1000_0000_0000_0000 or so + #undef PP_ODD #undef PP_ODD_H #undef PP_ODD_L @@ -137,15 +136,15 @@ __hexagon_muldf3: .falign .Lpossible_unf: - /* We end up with a positive exponent */ - /* But we may have rounded up to an exponent of 1. */ - /* If the exponent is 1, if we rounded up to it - * we need to also raise underflow - * Fortunately, this is pretty easy to detect, we must have +/- 0x0010_0000_0000_0000 - * And the PP should also have more than one bit set - */ - /* Note: ATMP should have abs(PP_HH) */ - /* Note: BTMPL should have 0x7FEFFFFF */ + // We end up with a positive exponent + // But we may have rounded up to an exponent of 1. + // If the exponent is 1, if we rounded up to it + // we need to also raise underflow + // Fortunately, this is pretty easy to detect, we must have +/- 0x0010_0000_0000_0000 + // And the PP should also have more than one bit set + // + // Note: ATMP should have abs(PP_HH) + // Note: BTMPL should have 0x7FEFFFFF { p0 = cmp.eq(AL,#0) p0 = bitsclr(AH,BTMPL) @@ -194,29 +193,25 @@ __hexagon_muldf3: BTMPH = sub(EXP0,BTMPH) TMP = #63 // max amount to shift } - /* Underflow */ - /* - * PP_HH has the partial product with sticky LSB. - * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so - * The exponent of PP_HH is in EXP1, which is non-positive (0 or negative) - * That's the exponent that happens after the normalization - * - * EXP0 has the exponent that, when added to the normalized value, is out of range. - * - * Strategy: - * - * * Shift down bits, with sticky bit, such that the bits are aligned according - * to the LZ count and appropriate exponent, but not all the way to mantissa - * field, keep around the last few bits. - * * Put a 1 near the MSB - * * Check the LSBs for inexact; if inexact also set underflow - * * Convert [u]d2df -- will correctly round according to rounding mode - * * Replace exponent field with zero - * - * - */ - + // Underflow + // + // PP_HH has the partial product with sticky LSB. + // PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x1000_0000_0000_0000 or so + // The exponent of PP_HH is in EXP1, which is non-positive (0 or negative) + // That's the exponent that happens after the normalization + // + // EXP0 has the exponent that, when added to the normalized value, is out of range. + // + // Strategy: + // + // * Shift down bits, with sticky bit, such that the bits are aligned according + // to the LZ count and appropriate exponent, but not all the way to mantissa + // field, keep around the last few bits. + // * Put a 1 near the MSB + // * Check the LSBs for inexact; if inexact also set underflow + // * Convert [u]d2df -- will correctly round according to rounding mode + // * Replace exponent field with zero { BTMPL = #0 // offset for extract diff --git a/lib/builtins/hexagon/dfsqrt.S b/lib/builtins/hexagon/dfsqrt.S index 027d9e1fd..f1435e868 100644 --- a/lib/builtins/hexagon/dfsqrt.S +++ b/lib/builtins/hexagon/dfsqrt.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision square root */ +// Double Precision square root #define EXP r28 @@ -169,9 +168,9 @@ __hexagon_sqrt: #define P_CARRY1 p2 #define P_CARRY2 p3 - /* Iteration 0 */ - /* Maybe we can save a cycle by starting with ERROR=asl(fracrad), then as we multiply */ - /* We can shift and subtract instead of shift and add? */ + // Iteration 0 + // Maybe we can save a cycle by starting with ERROR=asl(fracrad), then as we multiply + // We can shift and subtract instead of shift and add? { ERROR = asl(FRACRAD,#15) PROD = mpyu(ROOTHI,ROOTHI) @@ -194,7 +193,7 @@ __hexagon_sqrt: SHIFTAMT = add(SHIFTAMT,#16) ERROR = asl(FRACRAD,#31) // for next iter } - /* Iteration 1 */ + // Iteration 1 { PROD = mpyu(ROOTHI,ROOTHI) ERROR -= mpyu(ROOTHI,ROOTLO) // amount is 31, no shift needed @@ -214,7 +213,7 @@ __hexagon_sqrt: SHIFTAMT = add(SHIFTAMT,#16) ERROR = asl(FRACRAD,#47) // for next iter } - /* Iteration 2 */ + // Iteration 2 { PROD = mpyu(ROOTHI,ROOTHI) } @@ -245,7 +244,7 @@ __hexagon_sqrt: #undef RECIPEST #undef SHIFTAMT #define TWOROOT_LO r9:8 - /* Adjust Root */ + // Adjust Root { HL = mpyu(ROOTHI,ROOTLO) LL = mpyu(ROOTLO,ROOTLO) diff --git a/lib/builtins/hexagon/divdi3.S b/lib/builtins/hexagon/divdi3.S index 49ee8104f..770601a47 100644 --- a/lib/builtins/hexagon/divdi3.S +++ b/lib/builtins/hexagon/divdi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/divsi3.S b/lib/builtins/hexagon/divsi3.S index 8e159baa1..5f406524e 100644 --- a/lib/builtins/hexagon/divsi3.S +++ b/lib/builtins/hexagon/divsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fabs_opt.S b/lib/builtins/hexagon/fabs_opt.S index b09b00734..6bf9b84b3 100644 --- a/lib/builtins/hexagon/fabs_opt.S +++ b/lib/builtins/hexagon/fabs_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fastmath2_dlib_asm.S b/lib/builtins/hexagon/fastmath2_dlib_asm.S index 9286df06c..574a04432 100644 --- a/lib/builtins/hexagon/fastmath2_dlib_asm.S +++ b/lib/builtins/hexagon/fastmath2_dlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== */ diff --git a/lib/builtins/hexagon/fastmath2_ldlib_asm.S b/lib/builtins/hexagon/fastmath2_ldlib_asm.S index 419255535..cf623f94c 100644 --- a/lib/builtins/hexagon/fastmath2_ldlib_asm.S +++ b/lib/builtins/hexagon/fastmath2_ldlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== * diff --git a/lib/builtins/hexagon/fastmath_dlib_asm.S b/lib/builtins/hexagon/fastmath_dlib_asm.S index 215936b78..3e59526c1 100644 --- a/lib/builtins/hexagon/fastmath_dlib_asm.S +++ b/lib/builtins/hexagon/fastmath_dlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== */ diff --git a/lib/builtins/hexagon/fma_opt.S b/lib/builtins/hexagon/fma_opt.S index 12378f0da..7f566adff 100644 --- a/lib/builtins/hexagon/fma_opt.S +++ b/lib/builtins/hexagon/fma_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fmax_opt.S b/lib/builtins/hexagon/fmax_opt.S index f3a218c97..81d711dff 100644 --- a/lib/builtins/hexagon/fmax_opt.S +++ b/lib/builtins/hexagon/fmax_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fmin_opt.S b/lib/builtins/hexagon/fmin_opt.S index ef9b0ff85..d043f1d7a 100644 --- a/lib/builtins/hexagon/fmin_opt.S +++ b/lib/builtins/hexagon/fmin_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S index fbe09086c..10b81f653 100644 --- a/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S +++ b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/hexagon/memcpy_likely_aligned.S b/lib/builtins/hexagon/memcpy_likely_aligned.S index bbc85c22d..492298f10 100644 --- a/lib/builtins/hexagon/memcpy_likely_aligned.S +++ b/lib/builtins/hexagon/memcpy_likely_aligned.S @@ -1,9 +1,8 @@ //===------------------------- memcopy routines ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/moddi3.S b/lib/builtins/hexagon/moddi3.S index 12a0595fe..d4246b61b 100644 --- a/lib/builtins/hexagon/moddi3.S +++ b/lib/builtins/hexagon/moddi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/modsi3.S b/lib/builtins/hexagon/modsi3.S index 5afda9e29..4015d5e06 100644 --- a/lib/builtins/hexagon/modsi3.S +++ b/lib/builtins/hexagon/modsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/sfdiv_opt.S b/lib/builtins/hexagon/sfdiv_opt.S index 6bdd4808c..7c9ae14b7 100644 --- a/lib/builtins/hexagon/sfdiv_opt.S +++ b/lib/builtins/hexagon/sfdiv_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/sfsqrt_opt.S b/lib/builtins/hexagon/sfsqrt_opt.S index 7f6190027..532df9a06 100644 --- a/lib/builtins/hexagon/sfsqrt_opt.S +++ b/lib/builtins/hexagon/sfsqrt_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivdi3.S b/lib/builtins/hexagon/udivdi3.S index 1ca326b75..23f931d4f 100644 --- a/lib/builtins/hexagon/udivdi3.S +++ b/lib/builtins/hexagon/udivdi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivmoddi4.S b/lib/builtins/hexagon/udivmoddi4.S index deb5aae09..6dbfc59bd 100644 --- a/lib/builtins/hexagon/udivmoddi4.S +++ b/lib/builtins/hexagon/udivmoddi4.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivmodsi4.S b/lib/builtins/hexagon/udivmodsi4.S index 25bbe7cd5..9e231212d 100644 --- a/lib/builtins/hexagon/udivmodsi4.S +++ b/lib/builtins/hexagon/udivmodsi4.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivsi3.S b/lib/builtins/hexagon/udivsi3.S index 54f0aa409..d68599a8e 100644 --- a/lib/builtins/hexagon/udivsi3.S +++ b/lib/builtins/hexagon/udivsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/umoddi3.S b/lib/builtins/hexagon/umoddi3.S index f09152141..646ca128d 100644 --- a/lib/builtins/hexagon/umoddi3.S +++ b/lib/builtins/hexagon/umoddi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/umodsi3.S b/lib/builtins/hexagon/umodsi3.S index a8270c203..a92394486 100644 --- a/lib/builtins/hexagon/umodsi3.S +++ b/lib/builtins/hexagon/umodsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/i386/ashldi3.S b/lib/builtins/i386/ashldi3.S index 6f05dcf74..7ba912692 100644 --- a/lib/builtins/i386/ashldi3.S +++ b/lib/builtins/i386/ashldi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/ashrdi3.S b/lib/builtins/i386/ashrdi3.S index 206369f36..3cca4782a 100644 --- a/lib/builtins/i386/ashrdi3.S +++ b/lib/builtins/i386/ashrdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -22,10 +23,10 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) #endif psrlq %xmm2, %xmm0 // unsigned shift input by count - + testl %eax, %eax // check the sign-bit of the input jns 1f // early out for positive inputs - + // If the input is negative, we need to construct the shifted sign bit // to or into the result, as xmm does not have a signed right shift. pcmpeqb %xmm1, %xmm1 // -1ULL @@ -35,7 +36,7 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) psubq %xmm1, %xmm2 // 64 - count psllq %xmm2, %xmm1 // -1 << (64 - count) = leading sign bits por %xmm1, %xmm0 - + // Move the result back to the general purpose registers and return 1: movd %xmm0, %eax psrlq $32, %xmm0 @@ -51,14 +52,14 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) movl 12(%esp), %ecx // Load count movl 8(%esp), %edx // Load high movl 4(%esp), %eax // Load low - + testl $0x20, %ecx // If count >= 32 jnz 1f // goto 1 shrdl %cl, %edx, %eax // right shift low by count sarl %cl, %edx // right shift high by count ret - + 1: movl %edx, %eax // Move high to low sarl $31, %edx // clear high sarl %cl, %eax // shift low by count - 32 diff --git a/lib/builtins/i386/chkstk.S b/lib/builtins/i386/chkstk.S index b59974868..f0bea2187 100644 --- a/lib/builtins/i386/chkstk.S +++ b/lib/builtins/i386/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/chkstk2.S b/lib/builtins/i386/chkstk2.S index 7d65bb088..5d6cbdfa5 100644 --- a/lib/builtins/i386/chkstk2.S +++ b/lib/builtins/i386/chkstk2.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/divdi3.S b/lib/builtins/i386/divdi3.S index 2fb4bdcad..09e1e42eb 100644 --- a/lib/builtins/i386/divdi3.S +++ b/lib/builtins/i386/divdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -22,9 +23,9 @@ .balign 4 DEFINE_COMPILERRT_FUNCTION(__divdi3) -/* This is currently implemented by wrapping the unsigned divide up in an absolute - value, then restoring the correct sign at the end of the computation. This could - certainly be improved upon. */ +// This is currently implemented by wrapping the unsigned divide up in an absolute +// value, then restoring the correct sign at the end of the computation. This could +// certainly be improved upon. pushl %esi movl 20(%esp), %edx // high word of b @@ -38,7 +39,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl %edx, 20(%esp) movl %eax, 16(%esp) // store abs(b) back to stack movl %ecx, %esi // set aside sign of b - + movl 12(%esp), %edx // high word of b movl 8(%esp), %eax // low word of b movl %edx, %ecx @@ -55,11 +56,11 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 24(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -68,10 +69,10 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 16(%esp), %edx // Load the high and low words of a, and jump movl 12(%esp), %eax // to [1] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 1f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 1f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -90,7 +91,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) sbbl $0, %edi // decrement q if remainder is negative xorl %edx, %edx movl %edi, %eax - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -101,8 +102,8 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) retl // Return -1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +1: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -128,7 +129,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) sbbl $0, %edi // decrement q if remainder is negative xorl %edx, %edx movl %edi, %eax - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -138,8 +139,8 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) popl %esi retl // Return - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 16(%esp), %eax // Find qhi and rhi such that movl 20(%esp), %ecx // @@ -149,7 +150,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 12(%esp), %eax // Find qlo such that divl %ecx // movl %ebx, %edx // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax diff --git a/lib/builtins/i386/floatdidf.S b/lib/builtins/i386/floatdidf.S index d75dfe62d..ab7422c31 100644 --- a/lib/builtins/i386/floatdidf.S +++ b/lib/builtins/i386/floatdidf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatdisf.S b/lib/builtins/i386/floatdisf.S index 0874eaaa9..d91f14e9f 100644 --- a/lib/builtins/i386/floatdisf.S +++ b/lib/builtins/i386/floatdisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatdixf.S b/lib/builtins/i386/floatdixf.S index 1044ef55a..df70f5f9e 100644 --- a/lib/builtins/i386/floatdixf.S +++ b/lib/builtins/i386/floatdixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatundidf.S b/lib/builtins/i386/floatundidf.S index fe032348e..8b1b666ce 100644 --- a/lib/builtins/i386/floatundidf.S +++ b/lib/builtins/i386/floatundidf.S @@ -1,9 +1,8 @@ //===-- floatundidf.S - Implement __floatundidf for i386 ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/i386/floatundisf.S b/lib/builtins/i386/floatundisf.S index 16000b576..44301719e 100644 --- a/lib/builtins/i386/floatundisf.S +++ b/lib/builtins/i386/floatundisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -48,7 +49,7 @@ END_COMPILERRT_FUNCTION(__floatundisf) */ -/* branch-free, x87-free implementation - faster at the expense of code size */ +// branch-free, x87-free implementation - faster at the expense of code size #ifdef __i386__ @@ -78,7 +79,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundisf) movd 8(%esp), %xmm1 movd 4(%esp), %xmm0 punpckldq %xmm1, %xmm0 - + calll 0f 0: popl %ecx shrl %eax // high 31 bits of input as sint32 diff --git a/lib/builtins/i386/floatundixf.S b/lib/builtins/i386/floatundixf.S index c935670cb..30b4d9f4b 100644 --- a/lib/builtins/i386/floatundixf.S +++ b/lib/builtins/i386/floatundixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/lshrdi3.S b/lib/builtins/i386/lshrdi3.S index 53e95cf76..896633e85 100644 --- a/lib/builtins/i386/lshrdi3.S +++ b/lib/builtins/i386/lshrdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -41,14 +42,14 @@ DEFINE_COMPILERRT_FUNCTION(__lshrdi3) movl 12(%esp), %ecx // Load count movl 8(%esp), %edx // Load high movl 4(%esp), %eax // Load low - + testl $0x20, %ecx // If count >= 32 jnz 1f // goto 1 shrdl %cl, %edx, %eax // right shift low by count shrl %cl, %edx // right shift high by count ret - + 1: movl %edx, %eax // Move high to low xorl %edx, %edx // clear high shrl %cl, %eax // shift low by count - 32 diff --git a/lib/builtins/i386/moddi3.S b/lib/builtins/i386/moddi3.S index a5bf9ce8e..4580f20ea 100644 --- a/lib/builtins/i386/moddi3.S +++ b/lib/builtins/i386/moddi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -23,8 +24,8 @@ .balign 4 DEFINE_COMPILERRT_FUNCTION(__moddi3) -/* This is currently implemented by wrapping the unsigned modulus up in an absolute - value. This could certainly be improved upon. */ +// This is currently implemented by wrapping the unsigned modulus up in an absolute +// value. This could certainly be improved upon. pushl %esi movl 20(%esp), %edx // high word of b @@ -37,7 +38,7 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) sbbl %ecx, %edx // EDX:EAX = abs(b) movl %edx, 20(%esp) movl %eax, 16(%esp) // store abs(b) back to stack - + movl 12(%esp), %edx // high word of b movl 8(%esp), %eax // low word of b movl %edx, %ecx @@ -54,11 +55,11 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 24(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -67,10 +68,10 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 16(%esp), %edx // Load the high and low words of a, and jump movl 12(%esp), %eax // to [2] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 2f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 2f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -86,13 +87,13 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 28(%esp), %eax imull %edi, %eax // q*bhi subl %eax, %ecx // ECX:EBX = a - q*b - + jnc 1f // if positive, this is the result. addl 24(%esp), %ebx // otherwise adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 1: movl %ebx, %eax movl %ecx, %edx - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -102,8 +103,8 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) popl %esi retl // Return -2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +2: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -132,7 +133,7 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 3: movl %ebx, %eax movl %ecx, %edx - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -141,8 +142,8 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) popl %ebx popl %esi retl // Return - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 16(%esp), %eax // Find qhi and rhi such that movl 20(%esp), %ecx // diff --git a/lib/builtins/i386/muldi3.S b/lib/builtins/i386/muldi3.S index 123946064..a898e2414 100644 --- a/lib/builtins/i386/muldi3.S +++ b/lib/builtins/i386/muldi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -14,15 +15,15 @@ DEFINE_COMPILERRT_FUNCTION(__muldi3) movl 16(%esp), %eax // b.lo movl 12(%esp), %ecx // a.hi imull %eax, %ecx // b.lo * a.hi - + movl 8(%esp), %edx // a.lo movl 20(%esp), %ebx // b.hi imull %edx, %ebx // a.lo * b.hi - + mull %edx // EDX:EAX = a.lo * b.lo addl %ecx, %ebx // EBX = (a.lo*b.hi + a.hi*b.lo) addl %ebx, %edx - + popl %ebx retl END_COMPILERRT_FUNCTION(__muldi3) diff --git a/lib/builtins/i386/udivdi3.S b/lib/builtins/i386/udivdi3.S index 727613639..ca390245e 100644 --- a/lib/builtins/i386/udivdi3.S +++ b/lib/builtins/i386/udivdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -26,11 +27,11 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) movl 20(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -39,10 +40,10 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) movl 12(%esp), %edx // Load the high and low words of a, and jump movl 8(%esp), %eax // to [1] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 1f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 1f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -66,8 +67,8 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) retl -1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +1: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -97,8 +98,8 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) popl %ebx retl - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 12(%esp), %eax // Find qhi and rhi such that movl 16(%esp), %ecx // diff --git a/lib/builtins/i386/umoddi3.S b/lib/builtins/i386/umoddi3.S index 763e82194..2717e7e5d 100644 --- a/lib/builtins/i386/umoddi3.S +++ b/lib/builtins/i386/umoddi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -27,11 +28,11 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 20(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -40,10 +41,10 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 12(%esp), %edx // Load the high and low words of a, and jump movl 8(%esp), %eax // to [2] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 2f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 2f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -59,20 +60,20 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 24(%esp), %eax imull %edi, %eax // q*bhi subl %eax, %ecx // ECX:EBX = a - q*b - + jnc 1f // if positive, this is the result. addl 20(%esp), %ebx // otherwise adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 1: movl %ebx, %eax movl %ecx, %edx - + popl %edi popl %ebx retl -2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +2: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -101,14 +102,14 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 3: movl %ebx, %eax movl %ecx, %edx - + popl %edi popl %ebx retl - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 12(%esp), %eax // Find qhi and rhi such that movl 16(%esp), %ecx // @@ -120,7 +121,7 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl %edx, %eax // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b popl %ebx // xorl %edx, %edx // and return 0:rlo - retl // + retl // END_COMPILERRT_FUNCTION(__umoddi3) #endif // __i386__ diff --git a/lib/builtins/int_endianness.h b/lib/builtins/int_endianness.h index e2586c56b..def046c34 100644 --- a/lib/builtins/int_endianness.h +++ b/lib/builtins/int_endianness.h @@ -1,51 +1,49 @@ -/* ===-- int_endianness.h - configuration header for compiler-rt ------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is a configuration header for compiler-rt. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_endianness.h - configuration header for compiler-rt -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef INT_ENDIANNESS_H #define INT_ENDIANNESS_H -#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ defined(__ORDER_LITTLE_ENDIAN__) -/* Clang and GCC provide built-in endianness definitions. */ +// Clang and GCC provide built-in endianness definitions. #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* __BYTE_ORDER__ */ +#define _YUGA_BIG_ENDIAN 0 +#endif // __BYTE_ORDER__ -#else /* Compilers other than Clang or GCC. */ +#else // Compilers other than Clang or GCC. #if defined(__SVR4) && defined(__sun) #include <sys/byteorder.h> #if defined(_BIG_ENDIAN) #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif defined(_LITTLE_ENDIAN) #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#else /* !_LITTLE_ENDIAN */ +#define _YUGA_BIG_ENDIAN 0 +#else // !_LITTLE_ENDIAN #error "unknown endianness" -#endif /* !_LITTLE_ENDIAN */ +#endif // !_LITTLE_ENDIAN -#endif /* Solaris and AuroraUX. */ +#endif // Solaris and AuroraUX. -/* .. */ +// .. #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ defined(__minix) @@ -53,64 +51,64 @@ #if _BYTE_ORDER == _BIG_ENDIAN #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* _BYTE_ORDER */ +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER -#endif /* *BSD */ +#endif // *BSD #if defined(__OpenBSD__) #include <machine/endian.h> #if _BYTE_ORDER == _BIG_ENDIAN #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* _BYTE_ORDER */ +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER -#endif /* OpenBSD */ +#endif // OpenBSD -/* .. */ +// .. -/* Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the - * compiler (at least with GCC) */ -#if defined(__APPLE__) || defined(__ellcc__ ) +// Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the +// compiler (at least with GCC) +#if defined(__APPLE__) || defined(__ellcc__) #ifdef __BIG_ENDIAN__ #if __BIG_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #endif -#endif /* __BIG_ENDIAN__ */ +#endif // __BIG_ENDIAN__ #ifdef __LITTLE_ENDIAN__ #if __LITTLE_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 0 #endif -#endif /* __LITTLE_ENDIAN__ */ +#endif // __LITTLE_ENDIAN__ -#endif /* Mac OSX */ +#endif // Mac OSX -/* .. */ +// .. #if defined(_WIN32) #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 0 -#endif /* Windows */ +#endif // Windows -#endif /* Clang or GCC. */ +#endif // Clang or GCC. -/* . */ +// . #if !defined(_YUGA_LITTLE_ENDIAN) || !defined(_YUGA_BIG_ENDIAN) #error Unable to determine endian -#endif /* Check we found an endianness correctly. */ +#endif // Check we found an endianness correctly. -#endif /* INT_ENDIANNESS_H */ +#endif // INT_ENDIANNESS_H diff --git a/lib/builtins/int_lib.h b/lib/builtins/int_lib.h index fe8a3bded..3092f68c0 100644 --- a/lib/builtins/int_lib.h +++ b/lib/builtins/int_lib.h @@ -1,44 +1,33 @@ -/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is a configuration header for compiler-rt. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef INT_LIB_H #define INT_LIB_H -/* Assumption: Signed integral is 2's complement. */ -/* Assumption: Right shift of signed negative is arithmetic shift. */ -/* Assumption: Endianness is little or big (not mixed). */ +// Assumption: Signed integral is 2's complement. +// Assumption: Right shift of signed negative is arithmetic shift. +// Assumption: Endianness is little or big (not mixed). -#if defined(__ELF__) -#define FNALIAS(alias_name, original_name) \ - void alias_name() __attribute__((__alias__(#original_name))) -#define COMPILER_RT_ALIAS(aliasee) __attribute__((__alias__(#aliasee))) -#else -#define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") -#define COMPILER_RT_ALIAS(aliasee) _Pragma("GCC error(\"alias unsupported on this file format\")") -#endif - -/* ABI macro definitions */ +// ABI macro definitions #if __ARM_EABI__ -# ifdef COMPILER_RT_ARMHF_TARGET -# define COMPILER_RT_ABI -# else -# define COMPILER_RT_ABI __attribute__((__pcs__("aapcs"))) -# endif +#ifdef COMPILER_RT_ARMHF_TARGET +#define COMPILER_RT_ABI +#else +#define COMPILER_RT_ABI __attribute__((__pcs__("aapcs"))) +#endif #else -# define COMPILER_RT_ABI +#define COMPILER_RT_ABI #endif #define AEABI_RTABI __attribute__((__pcs__("aapcs"))) @@ -55,26 +44,44 @@ #define UNUSED __attribute__((unused)) #endif +#define STR(a) #a +#define XSTR(a) STR(a) +#define SYMBOL_NAME(name) XSTR(__USER_LABEL_PREFIX__) #name + +#if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__) +#define COMPILER_RT_ALIAS(name, aliasname) \ + COMPILER_RT_ABI __typeof(name) aliasname __attribute__((__alias__(#name))); +#elif defined(__APPLE__) +#define COMPILER_RT_ALIAS(name, aliasname) \ + __asm__(".globl " SYMBOL_NAME(aliasname)); \ + __asm__(SYMBOL_NAME(aliasname) " = " SYMBOL_NAME(name)); \ + COMPILER_RT_ABI __typeof(name) aliasname; +#elif defined(_WIN32) +#define COMPILER_RT_ALIAS(name, aliasname) +#else +#error Unsupported target +#endif + #if defined(__NetBSD__) && (defined(_KERNEL) || defined(_STANDALONE)) -/* - * Kernel and boot environment can't use normal headers, - * so use the equivalent system headers. - */ -# include <machine/limits.h> -# include <sys/stdint.h> -# include <sys/types.h> +// +// Kernel and boot environment can't use normal headers, +// so use the equivalent system headers. +// +#include <machine/limits.h> +#include <sys/stdint.h> +#include <sys/types.h> #else -/* Include the standard compiler builtin headers we use functionality from. */ -# include <limits.h> -# include <stdint.h> -# include <stdbool.h> -# include <float.h> +// Include the standard compiler builtin headers we use functionality from. +#include <float.h> +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> #endif -/* Include the commonly used internal type definitions. */ +// Include the commonly used internal type definitions. #include "int_types.h" -/* Include internal utility function declarations. */ +// Include internal utility function declarations. #include "int_util.h" COMPILER_RT_ABI si_int __paritysi2(si_int a); @@ -84,14 +91,14 @@ COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b); COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b); COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d); -COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int* rem); -COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int* rem); +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem); +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem); #ifdef CRT_HAS_128BIT COMPILER_RT_ABI si_int __clzti2(ti_int a); -COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int* rem); +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem); #endif -/* Definitions for builtins unavailable on MSVC */ +// Definitions for builtins unavailable on MSVC #if defined(_MSC_VER) && !defined(__clang__) #include <intrin.h> @@ -129,6 +136,6 @@ uint32_t __inline __builtin_clzll(uint64_t value) { #endif #define __builtin_clzl __builtin_clzll -#endif /* defined(_MSC_VER) && !defined(__clang__) */ +#endif // defined(_MSC_VER) && !defined(__clang__) -#endif /* INT_LIB_H */ +#endif // INT_LIB_H diff --git a/lib/builtins/int_math.h b/lib/builtins/int_math.h index aa3d0721a..58d8990f3 100644 --- a/lib/builtins/int_math.h +++ b/lib/builtins/int_math.h @@ -1,34 +1,31 @@ -/* ===-- int_math.h - internal math inlines ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===-----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines substitutes for the libm functions used in some of the - * compiler-rt implementations, defined in such a way that there is not a direct - * dependency on libm or math.h. Instead, we use the compiler builtin versions - * where available. This reduces our dependencies on the system SDK by foisting - * the responsibility onto the compiler. - * - * ===-----------------------------------------------------------------------=== - */ +//===-- int_math.h - internal math inlines --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines substitutes for the libm functions used in some of the +// compiler-rt implementations, defined in such a way that there is not a direct +// dependency on libm or math.h. Instead, we use the compiler builtin versions +// where available. This reduces our dependencies on the system SDK by foisting +// the responsibility onto the compiler. +// +//===----------------------------------------------------------------------===// #ifndef INT_MATH_H #define INT_MATH_H #ifndef __has_builtin -# define __has_builtin(x) 0 +#define __has_builtin(x) 0 #endif #if defined(_MSC_VER) && !defined(__clang__) #include <math.h> #include <stdlib.h> -#include <ymath.h> #endif #if defined(_MSC_VER) && !defined(__clang__) @@ -42,24 +39,23 @@ #define crt_isinf(x) !_finite((x)) #define crt_isnan(x) _isnan((x)) #else -/* Define crt_isfinite in terms of the builtin if available, otherwise provide - * an alternate version in terms of our other functions. This supports some - * versions of GCC which didn't have __builtin_isfinite. - */ +// Define crt_isfinite in terms of the builtin if available, otherwise provide +// an alternate version in terms of our other functions. This supports some +// versions of GCC which didn't have __builtin_isfinite. #if __has_builtin(__builtin_isfinite) -# define crt_isfinite(x) __builtin_isfinite((x)) +#define crt_isfinite(x) __builtin_isfinite((x)) #elif defined(__GNUC__) -# define crt_isfinite(x) \ - __extension__(({ \ - __typeof((x)) x_ = (x); \ - !crt_isinf(x_) && !crt_isnan(x_); \ - })) +#define crt_isfinite(x) \ + __extension__(({ \ + __typeof((x)) x_ = (x); \ + !crt_isinf(x_) && !crt_isnan(x_); \ + })) #else -# error "Do not know how to check for infinity" -#endif /* __has_builtin(__builtin_isfinite) */ +#error "Do not know how to check for infinity" +#endif // __has_builtin(__builtin_isfinite) #define crt_isinf(x) __builtin_isinf((x)) #define crt_isnan(x) __builtin_isnan((x)) -#endif /* _MSC_VER */ +#endif // _MSC_VER #if defined(_MSC_VER) && !defined(__clang__) #define crt_copysign(x, y) copysign((x), (y)) @@ -107,4 +103,4 @@ #define crt_scalbnl(x, y) __builtin_scalbnl((x), (y)) #endif -#endif /* INT_MATH_H */ +#endif // INT_MATH_H diff --git a/lib/builtins/int_types.h b/lib/builtins/int_types.h index 9f8da56cb..f89220d54 100644 --- a/lib/builtins/int_types.h +++ b/lib/builtins/int_types.h @@ -1,63 +1,57 @@ -/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines various standard types, most importantly a number of unions - * used to access parts of larger types. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines various standard types, most importantly a number of unions +// used to access parts of larger types. +// +//===----------------------------------------------------------------------===// #ifndef INT_TYPES_H #define INT_TYPES_H #include "int_endianness.h" -/* si_int is defined in Linux sysroot's asm-generic/siginfo.h */ +// si_int is defined in Linux sysroot's asm-generic/siginfo.h #ifdef si_int #undef si_int #endif -typedef int si_int; +typedef int si_int; typedef unsigned su_int; -typedef long long di_int; +typedef long long di_int; typedef unsigned long long du_int; -typedef union -{ - di_int all; - struct - { +typedef union { + di_int all; + struct { #if _YUGA_LITTLE_ENDIAN - su_int low; - si_int high; + su_int low; + si_int high; #else - si_int high; - su_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + si_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } dwords; -typedef union -{ - du_int all; - struct - { +typedef union { + du_int all; + struct { #if _YUGA_LITTLE_ENDIAN - su_int low; - su_int high; + su_int low; + su_int high; #else - su_int high; - su_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + su_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } udwords; #if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \ @@ -65,103 +59,93 @@ typedef union #define CRT_HAS_128BIT #endif -/* MSVC doesn't have a working 128bit integer type. Users should really compile - * compiler-rt with clang, but if they happen to be doing a standalone build for - * asan or something else, disable the 128 bit parts so things sort of work. - */ +// MSVC doesn't have a working 128bit integer type. Users should really compile +// compiler-rt with clang, but if they happen to be doing a standalone build for +// asan or something else, disable the 128 bit parts so things sort of work. #if defined(_MSC_VER) && !defined(__clang__) #undef CRT_HAS_128BIT #endif #ifdef CRT_HAS_128BIT -typedef int ti_int __attribute__ ((mode (TI))); -typedef unsigned tu_int __attribute__ ((mode (TI))); - -typedef union -{ - ti_int all; - struct - { +typedef int ti_int __attribute__((mode(TI))); +typedef unsigned tu_int __attribute__((mode(TI))); + +typedef union { + ti_int all; + struct { #if _YUGA_LITTLE_ENDIAN - du_int low; - di_int high; + du_int low; + di_int high; #else - di_int high; - du_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + di_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } twords; -typedef union -{ - tu_int all; - struct - { +typedef union { + tu_int all; + struct { #if _YUGA_LITTLE_ENDIAN - du_int low; - du_int high; + du_int low; + du_int high; #else - du_int high; - du_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + du_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } utwords; static __inline ti_int make_ti(di_int h, di_int l) { - twords r; - r.s.high = h; - r.s.low = l; - return r.all; + twords r; + r.s.high = h; + r.s.low = l; + return r.all; } static __inline tu_int make_tu(du_int h, du_int l) { - utwords r; - r.s.high = h; - r.s.low = l; - return r.all; + utwords r; + r.s.high = h; + r.s.low = l; + return r.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT -typedef union -{ - su_int u; - float f; +typedef union { + su_int u; + float f; } float_bits; -typedef union -{ - udwords u; - double f; +typedef union { + udwords u; + double f; } double_bits; -typedef struct -{ +typedef struct { #if _YUGA_LITTLE_ENDIAN - udwords low; - udwords high; + udwords low; + udwords high; #else - udwords high; - udwords low; -#endif /* _YUGA_LITTLE_ENDIAN */ + udwords high; + udwords low; +#endif // _YUGA_LITTLE_ENDIAN } uqwords; -/* Check if the target supports 80 bit extended precision long doubles. - * Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC - * still makes it 80 bits. Clang will match whatever compiler it is trying to - * be compatible with. - */ -#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ +// Check if the target supports 80 bit extended precision long doubles. +// Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC +// still makes it 80 bits. Clang will match whatever compiler it is trying to +// be compatible with. +#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ defined(__m68k__) || defined(__ia64__) #define HAS_80_BIT_LONG_DOUBLE 1 #else #define HAS_80_BIT_LONG_DOUBLE 0 #endif -typedef union -{ - uqwords u; - long double f; +typedef union { + uqwords u; + long double f; } long_double_bits; #if __STDC_VERSION__ >= 199901L @@ -172,14 +156,19 @@ typedef long double _Complex Lcomplex; #define COMPLEX_REAL(x) __real__(x) #define COMPLEX_IMAGINARY(x) __imag__(x) #else -typedef struct { float real, imaginary; } Fcomplex; +typedef struct { + float real, imaginary; +} Fcomplex; -typedef struct { double real, imaginary; } Dcomplex; +typedef struct { + double real, imaginary; +} Dcomplex; -typedef struct { long double real, imaginary; } Lcomplex; +typedef struct { + long double real, imaginary; +} Lcomplex; #define COMPLEX_REAL(x) (x).real #define COMPLEX_IMAGINARY(x) (x).imaginary #endif -#endif /* INT_TYPES_H */ - +#endif // INT_TYPES_H diff --git a/lib/builtins/int_util.c b/lib/builtins/int_util.c index 752f20155..226a6e934 100644 --- a/lib/builtins/int_util.c +++ b/lib/builtins/int_util.c @@ -1,25 +1,21 @@ -/* ===-- int_util.c - Implement internal utilities --------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_util.c - Implement internal utilities -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -#include "int_util.h" -/* NOTE: The definitions in this file are declared weak because we clients to be - * able to arbitrarily package individual functions into separate .a files. If - * we did not declare these weak, some link situations might end up seeing - * duplicate strong definitions of the same symbol. - * - * We can't use this solution for kernel use (which may not support weak), but - * currently expect that when built for kernel use all the functionality is - * packaged into a single library. - */ +// NOTE: The definitions in this file are declared weak because we clients to be +// able to arbitrarily package individual functions into separate .a files. If +// we did not declare these weak, some link situations might end up seeing +// duplicate strong definitions of the same symbol. +// +// We can't use this solution for kernel use (which may not support weak), but +// currently expect that when built for kernel use all the functionality is +// packaged into a single library. #ifdef KERNEL_USE @@ -33,7 +29,7 @@ void __compilerrt_abort_impl(const char *file, int line, const char *function) { #elif __APPLE__ -/* from libSystem.dylib */ +// from libSystem.dylib NORETURN extern void __assert_rtn(const char *func, const char *file, int line, const char *message); @@ -57,7 +53,7 @@ void __compilerrt_abort_impl(const char *file, int line, const char *function) { #else -/* Get the system definition of abort() */ +// Get the system definition of abort() #include <stdlib.h> #ifndef _WIN32 diff --git a/lib/builtins/int_util.h b/lib/builtins/int_util.h index c3c87381a..5fbdfb57c 100644 --- a/lib/builtins/int_util.h +++ b/lib/builtins/int_util.h @@ -1,25 +1,23 @@ -/* ===-- int_util.h - internal utility functions ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===-----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines non-inline utilities which are available for use in the - * library. The function definitions themselves are all contained in int_util.c - * which will always be compiled into any compiler-rt library. - * - * ===-----------------------------------------------------------------------=== - */ +//===-- int_util.h - internal utility functions ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines non-inline utilities which are available for use in the +// library. The function definitions themselves are all contained in int_util.c +// which will always be compiled into any compiler-rt library. +// +//===----------------------------------------------------------------------===// #ifndef INT_UTIL_H #define INT_UTIL_H -/** \brief Trigger a program abort (or panic for kernel code). */ +/// \brief Trigger a program abort (or panic for kernel code). #define compilerrt_abort() __compilerrt_abort_impl(__FILE__, __LINE__, __func__) NORETURN void __compilerrt_abort_impl(const char *file, int line, @@ -30,4 +28,4 @@ NORETURN void __compilerrt_abort_impl(const char *file, int line, #define COMPILE_TIME_ASSERT2(expr, cnt) \ typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED -#endif /* INT_UTIL_H */ +#endif // INT_UTIL_H diff --git a/lib/builtins/lshrdi3.c b/lib/builtins/lshrdi3.c index 67b2a7668..97e08e1e9 100644 --- a/lib/builtins/lshrdi3.c +++ b/lib/builtins/lshrdi3.c @@ -1,45 +1,38 @@ -/* ===-- lshrdi3.c - Implement __lshrdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __lshrdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- lshrdi3.c - Implement __lshrdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __lshrdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: logical a >> b */ +// Returns: logical a >> b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__lshrdi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - udwords input; - udwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - result.s.high = 0; - result.s.low = input.s.high >> (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI di_int __lshrdi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + udwords input; + udwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + result.s.high = 0; + result.s.low = input.s.high >> (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) COMPILER_RT_ALIAS(__lshrdi3); +COMPILER_RT_ALIAS(__lshrdi3, __aeabi_llsr) #endif diff --git a/lib/builtins/lshrti3.c b/lib/builtins/lshrti3.c index e4170ff84..d00a22095 100644 --- a/lib/builtins/lshrti3.c +++ b/lib/builtins/lshrti3.c @@ -1,45 +1,38 @@ -/* ===-- lshrti3.c - Implement __lshrti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __lshrti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- lshrti3.c - Implement __lshrti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __lshrti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: logical a >> b */ +// Returns: logical a >> b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__lshrti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - utwords input; - utwords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - result.s.high = 0; - result.s.low = input.s.high >> (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI ti_int __lshrti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + utwords input; + utwords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + result.s.high = 0; + result.s.low = input.s.high >> (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mingw_fixfloat.c b/lib/builtins/mingw_fixfloat.c index c462e0dbf..945be9d43 100644 --- a/lib/builtins/mingw_fixfloat.c +++ b/lib/builtins/mingw_fixfloat.c @@ -1,12 +1,10 @@ -/* ===-- mingw_fixfloat.c - Wrap int/float conversions for arm/windows -----=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mingw_fixfloat.c - Wrap int/float conversions for arm/windows -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" diff --git a/lib/builtins/moddi3.c b/lib/builtins/moddi3.c index a04279e38..92b099607 100644 --- a/lib/builtins/moddi3.c +++ b/lib/builtins/moddi3.c @@ -1,30 +1,26 @@ -/*===-- moddi3.c - Implement __moddi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __moddi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- moddi3.c - Implement __moddi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __moddi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI di_int -__moddi3(di_int a, di_int b) -{ - const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; - di_int s = b >> bits_in_dword_m1; /* s = b < 0 ? -1 : 0 */ - b = (b ^ s) - s; /* negate if s == -1 */ - s = a >> bits_in_dword_m1; /* s = a < 0 ? -1 : 0 */ - a = (a ^ s) - s; /* negate if s == -1 */ - du_int r; - __udivmoddi4(a, b, &r); - return ((di_int)r ^ s) - s; /* negate if s == -1 */ +COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) { + const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; + di_int s = b >> bits_in_dword_m1; // s = b < 0 ? -1 : 0 + b = (b ^ s) - s; // negate if s == -1 + s = a >> bits_in_dword_m1; // s = a < 0 ? -1 : 0 + a = (a ^ s) - s; // negate if s == -1 + du_int r; + __udivmoddi4(a, b, &r); + return ((di_int)r ^ s) - s; // negate if s == -1 } diff --git a/lib/builtins/modsi3.c b/lib/builtins/modsi3.c index 86c73ce13..e443b8a59 100644 --- a/lib/builtins/modsi3.c +++ b/lib/builtins/modsi3.c @@ -1,23 +1,19 @@ -/* ===-- modsi3.c - Implement __modsi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __modsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- modsi3.c - Implement __modsi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __modsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI si_int -__modsi3(si_int a, si_int b) -{ - return a - __divsi3(a, b) * b; +COMPILER_RT_ABI si_int __modsi3(si_int a, si_int b) { + return a - __divsi3(a, b) * b; } diff --git a/lib/builtins/modti3.c b/lib/builtins/modti3.c index d505c07ac..d11fe220b 100644 --- a/lib/builtins/modti3.c +++ b/lib/builtins/modti3.c @@ -1,34 +1,30 @@ -/* ===-- modti3.c - Implement __modti3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __modti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- modti3.c - Implement __modti3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __modti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/*Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI ti_int -__modti3(ti_int a, ti_int b) -{ - const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; - ti_int s = b >> bits_in_tword_m1; /* s = b < 0 ? -1 : 0 */ - b = (b ^ s) - s; /* negate if s == -1 */ - s = a >> bits_in_tword_m1; /* s = a < 0 ? -1 : 0 */ - a = (a ^ s) - s; /* negate if s == -1 */ - tu_int r; - __udivmodti4(a, b, &r); - return ((ti_int)r ^ s) - s; /* negate if s == -1 */ +COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) { + const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; + ti_int s = b >> bits_in_tword_m1; // s = b < 0 ? -1 : 0 + b = (b ^ s) - s; // negate if s == -1 + s = a >> bits_in_tword_m1; // s = a < 0 ? -1 : 0 + a = (a ^ s) - s; // negate if s == -1 + tu_int r; + __udivmodti4(a, b, &r); + return ((ti_int)r ^ s) - s; // negate if s == -1 } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/muldc3.c b/lib/builtins/muldc3.c index 16d8e9839..0ea7041c3 100644 --- a/lib/builtins/muldc3.c +++ b/lib/builtins/muldc3.c @@ -1,73 +1,65 @@ -/* ===-- muldc3.c - Implement __muldc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muldc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muldc3.c - Implement __muldc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muldc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Dcomplex -__muldc3(double __a, double __b, double __c, double __d) -{ - double __ac = __a * __c; - double __bd = __b * __d; - double __ad = __a * __d; - double __bc = __b * __c; - Dcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysign(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysign(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysign(0, __c); - if (crt_isnan(__d)) - __d = crt_copysign(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysign(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysign(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysign(0, __a); - if (crt_isnan(__b)) - __b = crt_copysign(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysign(0, __a); - if (crt_isnan(__b)) - __b = crt_copysign(0, __b); - if (crt_isnan(__c)) - __c = crt_copysign(0, __c); - if (crt_isnan(__d)) - __d = crt_copysign(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Dcomplex __muldc3(double __a, double __b, double __c, + double __d) { + double __ac = __a * __c; + double __bd = __b * __d; + double __ad = __a * __d; + double __bc = __b * __c; + Dcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysign(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysign(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysign(0, __c); + if (crt_isnan(__d)) + __d = crt_copysign(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysign(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysign(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysign(0, __a); + if (crt_isnan(__b)) + __b = crt_copysign(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysign(0, __a); + if (crt_isnan(__b)) + __b = crt_copysign(0, __b); + if (crt_isnan(__c)) + __c = crt_copysign(0, __c); + if (crt_isnan(__d)) + __d = crt_copysign(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } diff --git a/lib/builtins/muldf3.c b/lib/builtins/muldf3.c index 1bb103e38..f64b5228d 100644 --- a/lib/builtins/muldf3.c +++ b/lib/builtins/muldf3.c @@ -1,9 +1,8 @@ //===-- lib/muldf3.c - Double-precision multiplication ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define DOUBLE_PRECISION #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { - return __muldf3(a, b); -} +AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { return __muldf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__muldf3); +COMPILER_RT_ALIAS(__muldf3, __aeabi_dmul) #endif #endif diff --git a/lib/builtins/muldi3.c b/lib/builtins/muldi3.c index a187315e9..013f669a1 100644 --- a/lib/builtins/muldi3.c +++ b/lib/builtins/muldi3.c @@ -1,58 +1,51 @@ -/* ===-- muldi3.c - Implement __muldi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muldi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muldi3.c - Implement __muldi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muldi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -static -di_int -__muldsi3(su_int a, su_int b) -{ - dwords r; - const int bits_in_word_2 = (int)(sizeof(si_int) * CHAR_BIT) / 2; - const su_int lower_mask = (su_int)~0 >> bits_in_word_2; - r.s.low = (a & lower_mask) * (b & lower_mask); - su_int t = r.s.low >> bits_in_word_2; - r.s.low &= lower_mask; - t += (a >> bits_in_word_2) * (b & lower_mask); - r.s.low += (t & lower_mask) << bits_in_word_2; - r.s.high = t >> bits_in_word_2; - t = r.s.low >> bits_in_word_2; - r.s.low &= lower_mask; - t += (b >> bits_in_word_2) * (a & lower_mask); - r.s.low += (t & lower_mask) << bits_in_word_2; - r.s.high += t >> bits_in_word_2; - r.s.high += (a >> bits_in_word_2) * (b >> bits_in_word_2); - return r.all; +static di_int __muldsi3(su_int a, su_int b) { + dwords r; + const int bits_in_word_2 = (int)(sizeof(si_int) * CHAR_BIT) / 2; + const su_int lower_mask = (su_int)~0 >> bits_in_word_2; + r.s.low = (a & lower_mask) * (b & lower_mask); + su_int t = r.s.low >> bits_in_word_2; + r.s.low &= lower_mask; + t += (a >> bits_in_word_2) * (b & lower_mask); + r.s.low += (t & lower_mask) << bits_in_word_2; + r.s.high = t >> bits_in_word_2; + t = r.s.low >> bits_in_word_2; + r.s.low &= lower_mask; + t += (b >> bits_in_word_2) * (a & lower_mask); + r.s.low += (t & lower_mask) << bits_in_word_2; + r.s.high += t >> bits_in_word_2; + r.s.high += (a >> bits_in_word_2) * (b >> bits_in_word_2); + return r.all; } -/* Returns: a * b */ +// Returns: a * b -COMPILER_RT_ABI di_int -__muldi3(di_int a, di_int b) -{ - dwords x; - x.all = a; - dwords y; - y.all = b; - dwords r; - r.all = __muldsi3(x.s.low, y.s.low); - r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; - return r.all; +COMPILER_RT_ABI di_int __muldi3(di_int a, di_int b) { + dwords x; + x.all = a; + dwords y; + y.all = b; + dwords r; + r.all = __muldsi3(x.s.low, y.s.low); + r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; + return r.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) COMPILER_RT_ALIAS(__muldi3); +COMPILER_RT_ALIAS(__muldi3, __aeabi_lmul) #endif diff --git a/lib/builtins/mulodi4.c b/lib/builtins/mulodi4.c index d2fd7db2b..23f5571ac 100644 --- a/lib/builtins/mulodi4.c +++ b/lib/builtins/mulodi4.c @@ -1,58 +1,49 @@ -/*===-- mulodi4.c - Implement __mulodi4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulodi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulodi4.c - Implement __mulodi4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulodi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI di_int -__mulodi4(di_int a, di_int b, int* overflow) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - const di_int MIN = (di_int)1 << (N-1); - const di_int MAX = ~MIN; - *overflow = 0; - di_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - di_int sa = a >> (N - 1); - di_int abs_a = (a ^ sa) - sa; - di_int sb = b >> (N - 1); - di_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI di_int __mulodi4(di_int a, di_int b, int *overflow) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N - 1); + const di_int MAX = ~MIN; + *overflow = 0; + di_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } diff --git a/lib/builtins/mulosi4.c b/lib/builtins/mulosi4.c index 422528085..fea431129 100644 --- a/lib/builtins/mulosi4.c +++ b/lib/builtins/mulosi4.c @@ -1,58 +1,49 @@ -/*===-- mulosi4.c - Implement __mulosi4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulosi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulosi4.c - Implement __mulosi4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulosi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI si_int -__mulosi4(si_int a, si_int b, int* overflow) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - const si_int MIN = (si_int)1 << (N-1); - const si_int MAX = ~MIN; - *overflow = 0; - si_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - si_int sa = a >> (N - 1); - si_int abs_a = (a ^ sa) - sa; - si_int sb = b >> (N - 1); - si_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI si_int __mulosi4(si_int a, si_int b, int *overflow) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + const si_int MIN = (si_int)1 << (N - 1); + const si_int MAX = ~MIN; + *overflow = 0; + si_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + si_int sa = a >> (N - 1); + si_int abs_a = (a ^ sa) - sa; + si_int sb = b >> (N - 1); + si_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } diff --git a/lib/builtins/muloti4.c b/lib/builtins/muloti4.c index 16b218920..9bdd5b649 100644 --- a/lib/builtins/muloti4.c +++ b/lib/builtins/muloti4.c @@ -1,62 +1,53 @@ -/*===-- muloti4.c - Implement __muloti4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muloti4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muloti4.c - Implement __muloti4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muloti4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI ti_int -__muloti4(ti_int a, ti_int b, int* overflow) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - const ti_int MIN = (ti_int)1 << (N-1); - const ti_int MAX = ~MIN; - *overflow = 0; - ti_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - ti_int sa = a >> (N - 1); - ti_int abs_a = (a ^ sa) - sa; - ti_int sb = b >> (N - 1); - ti_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + const ti_int MIN = (ti_int)1 << (N - 1); + const ti_int MAX = ~MIN; + *overflow = 0; + ti_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + ti_int sa = a >> (N - 1); + ti_int abs_a = (a ^ sa) - sa; + ti_int sb = b >> (N - 1); + ti_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulsc3.c b/lib/builtins/mulsc3.c index c89cfd247..60653174e 100644 --- a/lib/builtins/mulsc3.c +++ b/lib/builtins/mulsc3.c @@ -1,73 +1,64 @@ -/* ===-- mulsc3.c - Implement __mulsc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulsc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulsc3.c - Implement __mulsc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulsc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Fcomplex -__mulsc3(float __a, float __b, float __c, float __d) -{ - float __ac = __a * __c; - float __bd = __b * __d; - float __ad = __a * __d; - float __bc = __b * __c; - Fcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysignf(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignf(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysignf(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignf(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysignf(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignf(0, __b); - if (crt_isnan(__c)) - __c = crt_copysignf(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignf(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Fcomplex __mulsc3(float __a, float __b, float __c, float __d) { + float __ac = __a * __c; + float __bd = __b * __d; + float __ad = __a * __d; + float __bc = __b * __c; + Fcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysignf(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignf(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysignf(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignf(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysignf(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignf(0, __b); + if (crt_isnan(__c)) + __c = crt_copysignf(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignf(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } diff --git a/lib/builtins/mulsf3.c b/lib/builtins/mulsf3.c index 1e2cf3e71..b9cf39abc 100644 --- a/lib/builtins/mulsf3.c +++ b/lib/builtins/mulsf3.c @@ -1,9 +1,8 @@ //===-- lib/mulsf3.c - Single-precision multiplication ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define SINGLE_PRECISION #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { - return __mulsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { return __mulsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__mulsf3); +COMPILER_RT_ALIAS(__mulsf3, __aeabi_fmul) #endif #endif diff --git a/lib/builtins/multc3.c b/lib/builtins/multc3.c index 0518bc256..bb7f6aabf 100644 --- a/lib/builtins/multc3.c +++ b/lib/builtins/multc3.c @@ -1,68 +1,65 @@ -/* ===-- multc3.c - Implement __multc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __multc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- multc3.c - Implement __multc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __multc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI long double _Complex -__multc3(long double a, long double b, long double c, long double d) -{ - long double ac = a * c; - long double bd = b * d; - long double ad = a * d; - long double bc = b * c; - long double _Complex z; - __real__ z = ac - bd; - __imag__ z = ad + bc; - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) { - int recalc = 0; - if (crt_isinf(a) || crt_isinf(b)) { - a = crt_copysignl(crt_isinf(a) ? 1 : 0, a); - b = crt_copysignl(crt_isinf(b) ? 1 : 0, b); - if (crt_isnan(c)) - c = crt_copysignl(0, c); - if (crt_isnan(d)) - d = crt_copysignl(0, d); - recalc = 1; - } - if (crt_isinf(c) || crt_isinf(d)) { - c = crt_copysignl(crt_isinf(c) ? 1 : 0, c); - d = crt_copysignl(crt_isinf(d) ? 1 : 0, d); - if (crt_isnan(a)) - a = crt_copysignl(0, a); - if (crt_isnan(b)) - b = crt_copysignl(0, b); - recalc = 1; - } - if (!recalc && (crt_isinf(ac) || crt_isinf(bd) || - crt_isinf(ad) || crt_isinf(bc))) { - if (crt_isnan(a)) - a = crt_copysignl(0, a); - if (crt_isnan(b)) - b = crt_copysignl(0, b); - if (crt_isnan(c)) - c = crt_copysignl(0, c); - if (crt_isnan(d)) - d = crt_copysignl(0, d); - recalc = 1; - } - if (recalc) { - __real__ z = CRT_INFINITY * (a * c - b * d); - __imag__ z = CRT_INFINITY * (a * d + b * c); - } +COMPILER_RT_ABI long double _Complex __multc3(long double a, long double b, + long double c, long double d) { + long double ac = a * c; + long double bd = b * d; + long double ad = a * d; + long double bc = b * c; + long double _Complex z; + __real__ z = ac - bd; + __imag__ z = ad + bc; + if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) { + int recalc = 0; + if (crt_isinf(a) || crt_isinf(b)) { + a = crt_copysignl(crt_isinf(a) ? 1 : 0, a); + b = crt_copysignl(crt_isinf(b) ? 1 : 0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; } - return z; + if (crt_isinf(c) || crt_isinf(d)) { + c = crt_copysignl(crt_isinf(c) ? 1 : 0, c); + d = crt_copysignl(crt_isinf(d) ? 1 : 0, d); + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + recalc = 1; + } + if (!recalc && + (crt_isinf(ac) || crt_isinf(bd) || crt_isinf(ad) || crt_isinf(bc))) { + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; + } + if (recalc) { + __real__ z = CRT_INFINITY * (a * c - b * d); + __imag__ z = CRT_INFINITY * (a * d + b * c); + } + } + return z; } diff --git a/lib/builtins/multf3.c b/lib/builtins/multf3.c index 0b915923e..0626fb8c7 100644 --- a/lib/builtins/multf3.c +++ b/lib/builtins/multf3.c @@ -1,9 +1,8 @@ //===-- lib/multf3.c - Quad-precision multiplication --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,6 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #endif diff --git a/lib/builtins/multi3.c b/lib/builtins/multi3.c index e0d52d430..d9d8b59cd 100644 --- a/lib/builtins/multi3.c +++ b/lib/builtins/multi3.c @@ -1,58 +1,51 @@ -/* ===-- multi3.c - Implement __multi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - - * This file implements __multi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- multi3.c - Implement __multi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __multi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ - -static -ti_int -__mulddi3(du_int a, du_int b) -{ - twords r; - const int bits_in_dword_2 = (int)(sizeof(di_int) * CHAR_BIT) / 2; - const du_int lower_mask = (du_int)~0 >> bits_in_dword_2; - r.s.low = (a & lower_mask) * (b & lower_mask); - du_int t = r.s.low >> bits_in_dword_2; - r.s.low &= lower_mask; - t += (a >> bits_in_dword_2) * (b & lower_mask); - r.s.low += (t & lower_mask) << bits_in_dword_2; - r.s.high = t >> bits_in_dword_2; - t = r.s.low >> bits_in_dword_2; - r.s.low &= lower_mask; - t += (b >> bits_in_dword_2) * (a & lower_mask); - r.s.low += (t & lower_mask) << bits_in_dword_2; - r.s.high += t >> bits_in_dword_2; - r.s.high += (a >> bits_in_dword_2) * (b >> bits_in_dword_2); - return r.all; +// Returns: a * b + +static ti_int __mulddi3(du_int a, du_int b) { + twords r; + const int bits_in_dword_2 = (int)(sizeof(di_int) * CHAR_BIT) / 2; + const du_int lower_mask = (du_int)~0 >> bits_in_dword_2; + r.s.low = (a & lower_mask) * (b & lower_mask); + du_int t = r.s.low >> bits_in_dword_2; + r.s.low &= lower_mask; + t += (a >> bits_in_dword_2) * (b & lower_mask); + r.s.low += (t & lower_mask) << bits_in_dword_2; + r.s.high = t >> bits_in_dword_2; + t = r.s.low >> bits_in_dword_2; + r.s.low &= lower_mask; + t += (b >> bits_in_dword_2) * (a & lower_mask); + r.s.low += (t & lower_mask) << bits_in_dword_2; + r.s.high += t >> bits_in_dword_2; + r.s.high += (a >> bits_in_dword_2) * (b >> bits_in_dword_2); + return r.all; } -/* Returns: a * b */ - -COMPILER_RT_ABI ti_int -__multi3(ti_int a, ti_int b) -{ - twords x; - x.all = a; - twords y; - y.all = b; - twords r; - r.all = __mulddi3(x.s.low, y.s.low); - r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; - return r.all; +// Returns: a * b + +COMPILER_RT_ABI ti_int __multi3(ti_int a, ti_int b) { + twords x; + x.all = a; + twords y; + y.all = b; + twords r; + r.all = __mulddi3(x.s.low, y.s.low); + r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; + return r.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulvdi3.c b/lib/builtins/mulvdi3.c index e63249e0a..cecc97ccf 100644 --- a/lib/builtins/mulvdi3.c +++ b/lib/builtins/mulvdi3.c @@ -1,56 +1,47 @@ -/*===-- mulvdi3.c - Implement __mulvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvdi3.c - Implement __mulvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI di_int -__mulvdi3(di_int a, di_int b) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - const di_int MIN = (di_int)1 << (N-1); - const di_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - di_int sa = a >> (N - 1); - di_int abs_a = (a ^ sa) - sa; - di_int sb = b >> (N - 1); - di_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N - 1); + const di_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } diff --git a/lib/builtins/mulvsi3.c b/lib/builtins/mulvsi3.c index 74ea4f2da..0d6b18ad0 100644 --- a/lib/builtins/mulvsi3.c +++ b/lib/builtins/mulvsi3.c @@ -1,56 +1,47 @@ -/* ===-- mulvsi3.c - Implement __mulvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvsi3.c - Implement __mulvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI si_int -__mulvsi3(si_int a, si_int b) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - const si_int MIN = (si_int)1 << (N-1); - const si_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - si_int sa = a >> (N - 1); - si_int abs_a = (a ^ sa) - sa; - si_int sb = b >> (N - 1); - si_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + const si_int MIN = (si_int)1 << (N - 1); + const si_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + si_int sa = a >> (N - 1); + si_int abs_a = (a ^ sa) - sa; + si_int sb = b >> (N - 1); + si_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } diff --git a/lib/builtins/mulvti3.c b/lib/builtins/mulvti3.c index f4c7d1612..03963a0ca 100644 --- a/lib/builtins/mulvti3.c +++ b/lib/builtins/mulvti3.c @@ -1,60 +1,51 @@ -/* ===-- mulvti3.c - Implement __mulvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvti3.c - Implement __mulvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI ti_int -__mulvti3(ti_int a, ti_int b) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - const ti_int MIN = (ti_int)1 << (N-1); - const ti_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - ti_int sa = a >> (N - 1); - ti_int abs_a = (a ^ sa) - sa; - ti_int sb = b >> (N - 1); - ti_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + const ti_int MIN = (ti_int)1 << (N - 1); + const ti_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + ti_int sa = a >> (N - 1); + ti_int abs_a = (a ^ sa) - sa; + ti_int sb = b >> (N - 1); + ti_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulxc3.c b/lib/builtins/mulxc3.c index ba3221691..2f7f14c28 100644 --- a/lib/builtins/mulxc3.c +++ b/lib/builtins/mulxc3.c @@ -1,77 +1,69 @@ -/* ===-- mulxc3.c - Implement __mulxc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulxc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulxc3.c - Implement __mulxc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulxc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Lcomplex -__mulxc3(long double __a, long double __b, long double __c, long double __d) -{ - long double __ac = __a * __c; - long double __bd = __b * __d; - long double __ad = __a * __d; - long double __bc = __b * __c; - Lcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysignl(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignl(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysignl(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignl(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysignl(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignl(0, __b); - if (crt_isnan(__c)) - __c = crt_copysignl(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignl(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Lcomplex __mulxc3(long double __a, long double __b, + long double __c, long double __d) { + long double __ac = __a * __c; + long double __bd = __b * __d; + long double __ad = __a * __d; + long double __bc = __b * __c; + Lcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysignl(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignl(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysignl(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignl(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysignl(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignl(0, __b); + if (crt_isnan(__c)) + __c = crt_copysignl(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignl(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } #endif diff --git a/lib/builtins/negdf2.c b/lib/builtins/negdf2.c index f0bfaad24..f9ceaa374 100644 --- a/lib/builtins/negdf2.c +++ b/lib/builtins/negdf2.c @@ -1,9 +1,8 @@ //===-- lib/negdf2.c - double-precision negation ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,17 +13,12 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__negdf2(fp_t a) { - return fromRep(toRep(a) ^ signBit); -} +COMPILER_RT_ABI fp_t __negdf2(fp_t a) { return fromRep(toRep(a) ^ signBit); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { - return __negdf2(a); -} +AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { return __negdf2(a); } #else -AEABI_RTABI fp_t __aeabi_dneg(fp_t a) COMPILER_RT_ALIAS(__negdf2); +COMPILER_RT_ALIAS(__negdf2, __aeabi_dneg) #endif #endif diff --git a/lib/builtins/negdi2.c b/lib/builtins/negdi2.c index 3d49ba289..5a525d4b0 100644 --- a/lib/builtins/negdi2.c +++ b/lib/builtins/negdi2.c @@ -1,26 +1,21 @@ -/* ===-- negdi2.c - Implement __negdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negdi2.c - Implement __negdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -COMPILER_RT_ABI di_int -__negdi2(di_int a) -{ - /* Note: this routine is here for API compatibility; any sane compiler - * should expand it inline. - */ - return -a; +COMPILER_RT_ABI di_int __negdi2(di_int a) { + // Note: this routine is here for API compatibility; any sane compiler + // should expand it inline. + return -a; } diff --git a/lib/builtins/negsf2.c b/lib/builtins/negsf2.c index 05c97d4d5..d59dfe7cf 100644 --- a/lib/builtins/negsf2.c +++ b/lib/builtins/negsf2.c @@ -1,9 +1,8 @@ //===-- lib/negsf2.c - single-precision negation ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,17 +13,12 @@ #define SINGLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__negsf2(fp_t a) { - return fromRep(toRep(a) ^ signBit); -} +COMPILER_RT_ABI fp_t __negsf2(fp_t a) { return fromRep(toRep(a) ^ signBit); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { - return __negsf2(a); -} +AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { return __negsf2(a); } #else -AEABI_RTABI fp_t __aeabi_fneg(fp_t a) COMPILER_RT_ALIAS(__negsf2); +COMPILER_RT_ALIAS(__negsf2, __aeabi_fneg) #endif #endif diff --git a/lib/builtins/negti2.c b/lib/builtins/negti2.c index 9b00b303f..d52ba4e13 100644 --- a/lib/builtins/negti2.c +++ b/lib/builtins/negti2.c @@ -1,30 +1,25 @@ -/* ===-- negti2.c - Implement __negti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negti2.c - Implement __negti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: -a */ +// Returns: -a -COMPILER_RT_ABI ti_int -__negti2(ti_int a) -{ - /* Note: this routine is here for API compatibility; any sane compiler - * should expand it inline. - */ - return -a; +COMPILER_RT_ABI ti_int __negti2(ti_int a) { + // Note: this routine is here for API compatibility; any sane compiler + // should expand it inline. + return -a; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/negvdi2.c b/lib/builtins/negvdi2.c index e336ecf28..5c52b3ec2 100644 --- a/lib/builtins/negvdi2.c +++ b/lib/builtins/negvdi2.c @@ -1,28 +1,24 @@ -/* ===-- negvdi2.c - Implement __negvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negvdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negvdi2.c - Implement __negvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI di_int -__negvdi2(di_int a) -{ - const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI di_int __negvdi2(di_int a) { + const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } diff --git a/lib/builtins/negvsi2.c b/lib/builtins/negvsi2.c index b9e93fef0..cccdee6dc 100644 --- a/lib/builtins/negvsi2.c +++ b/lib/builtins/negvsi2.c @@ -1,28 +1,24 @@ -/* ===-- negvsi2.c - Implement __negvsi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negvsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negvsi2.c - Implement __negvsi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI si_int -__negvsi2(si_int a) -{ - const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI si_int __negvsi2(si_int a) { + const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } diff --git a/lib/builtins/negvti2.c b/lib/builtins/negvti2.c index 85f9f7d19..8f92e1046 100644 --- a/lib/builtins/negvti2.c +++ b/lib/builtins/negvti2.c @@ -1,32 +1,28 @@ -/*===-- negvti2.c - Implement __negvti2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - *This file implements __negvti2 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- negvti2.c - Implement __negvti2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI ti_int -__negvti2(ti_int a) -{ - const ti_int MIN = (ti_int)1 << ((int)(sizeof(ti_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI ti_int __negvti2(ti_int a) { + const ti_int MIN = (ti_int)1 << ((int)(sizeof(ti_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/os_version_check.c b/lib/builtins/os_version_check.c index e0d40edc7..3794b9794 100644 --- a/lib/builtins/os_version_check.c +++ b/lib/builtins/os_version_check.c @@ -1,17 +1,15 @@ -/* ===-- os_version_check.c - OS version checking -------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements the function __isOSVersionAtLeast, used by - * Objective-C's @available - * - * ===----------------------------------------------------------------------=== - */ +//===-- os_version_check.c - OS version checking -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the function __isOSVersionAtLeast, used by +// Objective-C's @available +// +//===----------------------------------------------------------------------===// #ifdef __APPLE__ @@ -23,15 +21,15 @@ #include <stdlib.h> #include <string.h> -/* These three variables hold the host's OS version. */ +// These three variables hold the host's OS version. static int32_t GlobalMajor, GlobalMinor, GlobalSubminor; static dispatch_once_t DispatchOnceCounter; -/* We can't include <CoreFoundation/CoreFoundation.h> directly from here, so - * just forward declare everything that we need from it. */ +// We can't include <CoreFoundation/CoreFoundation.h> directly from here, so +// just forward declare everything that we need from it. typedef const void *CFDataRef, *CFAllocatorRef, *CFPropertyListRef, - *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; + *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; #if __LLP64__ typedef unsigned long long CFTypeID; @@ -48,9 +46,9 @@ typedef _Bool Boolean; typedef CFIndex CFPropertyListFormat; typedef uint32_t CFStringEncoding; -/* kCFStringEncodingASCII analog. */ +// kCFStringEncodingASCII analog. #define CF_STRING_ENCODING_ASCII 0x0600 -/* kCFStringEncodingUTF8 analog. */ +// kCFStringEncodingUTF8 analog. #define CF_STRING_ENCODING_UTF8 0x08000100 #define CF_PROPERTY_LIST_IMMUTABLE 0 @@ -74,10 +72,10 @@ typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex, CFStringEncoding); typedef void (*CFReleaseFuncTy)(CFTypeRef); -/* Find and parse the SystemVersion.plist file. */ +// Find and parse the SystemVersion.plist file. static void parseSystemVersionPList(void *Unused) { (void)Unused; - /* Load CoreFoundation dynamically */ + // Load CoreFoundation dynamically const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull"); if (!NullAllocator) return; @@ -88,18 +86,18 @@ static void parseSystemVersionPList(void *Unused) { if (!CFDataCreateWithBytesNoCopyFunc) return; CFPropertyListCreateWithDataFuncTy CFPropertyListCreateWithDataFunc = - (CFPropertyListCreateWithDataFuncTy)dlsym( - RTLD_DEFAULT, "CFPropertyListCreateWithData"); -/* CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it - * will be NULL on earlier OS versions. */ + (CFPropertyListCreateWithDataFuncTy)dlsym(RTLD_DEFAULT, + "CFPropertyListCreateWithData"); +// CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it +// will be NULL on earlier OS versions. #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" CFPropertyListCreateFromXMLDataFuncTy CFPropertyListCreateFromXMLDataFunc = (CFPropertyListCreateFromXMLDataFuncTy)dlsym( RTLD_DEFAULT, "CFPropertyListCreateFromXMLData"); #pragma clang diagnostic pop - /* CFPropertyListCreateFromXMLDataFunc is deprecated in macOS 10.10, so it - * might be NULL in future OS versions. */ + // CFPropertyListCreateFromXMLDataFunc is deprecated in macOS 10.10, so it + // might be NULL in future OS versions. if (!CFPropertyListCreateWithDataFunc && !CFPropertyListCreateFromXMLDataFunc) return; CFStringCreateWithCStringNoCopyFuncTy CFStringCreateWithCStringNoCopyFunc = @@ -143,7 +141,7 @@ static void parseSystemVersionPList(void *Unused) { if (!PropertyList) return; - /* Dynamically allocated stuff. */ + // Dynamically allocated stuff. CFDictionaryRef PListRef = NULL; CFDataRef FileContentsRef = NULL; UInt8 *PListBuf = NULL; @@ -162,8 +160,8 @@ static void parseSystemVersionPList(void *Unused) { if (NumRead != (size_t)PListFileSize) goto Fail; - /* Get the file buffer into CF's format. We pass in a null allocator here * - * because we free PListBuf ourselves */ + // Get the file buffer into CF's format. We pass in a null allocator here * + // because we free PListBuf ourselves FileContentsRef = (*CFDataCreateWithBytesNoCopyFunc)( NULL, PListBuf, (CFIndex)NumRead, AllocatorNull); if (!FileContentsRef) @@ -204,7 +202,7 @@ Fail: } int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) { - /* Populate the global version variables, if they haven't already. */ + // Populate the global version variables, if they haven't already. dispatch_once_f(&DispatchOnceCounter, NULL, parseSystemVersionPList); if (Major < GlobalMajor) @@ -220,7 +218,7 @@ int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) { #else -/* Silence an empty translation unit warning. */ +// Silence an empty translation unit warning. typedef int unused; #endif diff --git a/lib/builtins/paritydi2.c b/lib/builtins/paritydi2.c index 8ea5ab421..dd9d45e63 100644 --- a/lib/builtins/paritydi2.c +++ b/lib/builtins/paritydi2.c @@ -1,25 +1,21 @@ -/* ===-- paritydi2.c - Implement __paritydi2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __paritydi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- paritydi2.c - Implement __paritydi2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __paritydi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__paritydi2(di_int a) -{ - dwords x; - x.all = a; - return __paritysi2(x.s.high ^ x.s.low); +COMPILER_RT_ABI si_int __paritydi2(di_int a) { + dwords x; + x.all = a; + return __paritysi2(x.s.high ^ x.s.low); } diff --git a/lib/builtins/paritysi2.c b/lib/builtins/paritysi2.c index 599984663..3efa961f2 100644 --- a/lib/builtins/paritysi2.c +++ b/lib/builtins/paritysi2.c @@ -1,27 +1,23 @@ -/* ===-- paritysi2.c - Implement __paritysi2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __paritysi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- paritysi2.c - Implement __paritysi2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __paritysi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__paritysi2(si_int a) -{ - su_int x = (su_int)a; - x ^= x >> 16; - x ^= x >> 8; - x ^= x >> 4; - return (0x6996 >> (x & 0xF)) & 1; +COMPILER_RT_ABI si_int __paritysi2(si_int a) { + su_int x = (su_int)a; + x ^= x >> 16; + x ^= x >> 8; + x ^= x >> 4; + return (0x6996 >> (x & 0xF)) & 1; } diff --git a/lib/builtins/parityti2.c b/lib/builtins/parityti2.c index 5a4fe4924..f3942ba83 100644 --- a/lib/builtins/parityti2.c +++ b/lib/builtins/parityti2.c @@ -1,29 +1,25 @@ -/* ===-- parityti2.c - Implement __parityti2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __parityti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- parityti2.c - Implement __parityti2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __parityti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__parityti2(ti_int a) -{ - twords x; - x.all = a; - return __paritydi2(x.s.high ^ x.s.low); +COMPILER_RT_ABI si_int __parityti2(ti_int a) { + twords x; + x.all = a; + return __paritydi2(x.s.high ^ x.s.low); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/popcountdi2.c b/lib/builtins/popcountdi2.c index 5e8a62f07..9bbc39c66 100644 --- a/lib/builtins/popcountdi2.c +++ b/lib/builtins/popcountdi2.c @@ -1,36 +1,32 @@ -/* ===-- popcountdi2.c - Implement __popcountdi2 ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountdi2.c - Implement __popcountdi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountdi2(di_int a) -{ - du_int x2 = (du_int)a; - x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL); - /* Every 2 bits holds the sum of every pair of bits (32) */ - x2 = ((x2 >> 2) & 0x3333333333333333uLL) + (x2 & 0x3333333333333333uLL); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (16) */ - x2 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FuLL; - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (8) */ - su_int x = (su_int)(x2 + (x2 >> 32)); - /* The lower 32 bits hold four 16 bit sums (5 significant bits). */ - /* Upper 32 bits are garbage */ - x = x + (x >> 16); - /* The lower 16 bits hold two 32 bit sums (6 significant bits). */ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0x0000007F; /* (7 significant bits) */ +COMPILER_RT_ABI si_int __popcountdi2(di_int a) { + du_int x2 = (du_int)a; + x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL); + // Every 2 bits holds the sum of every pair of bits (32) + x2 = ((x2 >> 2) & 0x3333333333333333uLL) + (x2 & 0x3333333333333333uLL); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (16) + x2 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FuLL; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (8) + su_int x = (su_int)(x2 + (x2 >> 32)); + // The lower 32 bits hold four 16 bit sums (5 significant bits). + // Upper 32 bits are garbage + x = x + (x >> 16); + // The lower 16 bits hold two 32 bit sums (6 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000007F; // (7 significant bits) } diff --git a/lib/builtins/popcountsi2.c b/lib/builtins/popcountsi2.c index 44544ff49..75e592a77 100644 --- a/lib/builtins/popcountsi2.c +++ b/lib/builtins/popcountsi2.c @@ -1,33 +1,29 @@ -/* ===-- popcountsi2.c - Implement __popcountsi2 ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountsi2.c - Implement __popcountsi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountsi2(si_int a) -{ - su_int x = (su_int)a; - x = x - ((x >> 1) & 0x55555555); - /* Every 2 bits holds the sum of every pair of bits */ - x = ((x >> 2) & 0x33333333) + (x & 0x33333333); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) */ - x = (x + (x >> 4)) & 0x0F0F0F0F; - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) */ - x = (x + (x >> 16)); - /* The lower 16 bits hold two 8 bit sums (5 significant bits).*/ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0x0000003F; /* (6 significant bits) */ +COMPILER_RT_ABI si_int __popcountsi2(si_int a) { + su_int x = (su_int)a; + x = x - ((x >> 1) & 0x55555555); + // Every 2 bits holds the sum of every pair of bits + x = ((x >> 2) & 0x33333333) + (x & 0x33333333); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) + x = (x + (x >> 4)) & 0x0F0F0F0F; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) + x = (x + (x >> 16)); + // The lower 16 bits hold two 8 bit sums (5 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000003F; // (6 significant bits) } diff --git a/lib/builtins/popcountti2.c b/lib/builtins/popcountti2.c index 7451bbb28..853fd7223 100644 --- a/lib/builtins/popcountti2.c +++ b/lib/builtins/popcountti2.c @@ -1,44 +1,43 @@ -/* ===-- popcountti2.c - Implement __popcountti2 ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountti2.c - Implement __popcountti2 +//----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountti2(ti_int a) -{ - tu_int x3 = (tu_int)a; - x3 = x3 - ((x3 >> 1) & (((tu_int)0x5555555555555555uLL << 64) | - 0x5555555555555555uLL)); - /* Every 2 bits holds the sum of every pair of bits (64) */ - x3 = ((x3 >> 2) & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)) - + (x3 & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (32) */ - x3 = (x3 + (x3 >> 4)) - & (((tu_int)0x0F0F0F0F0F0F0F0FuLL << 64) | 0x0F0F0F0F0F0F0F0FuLL); - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (16) */ - du_int x2 = (du_int)(x3 + (x3 >> 64)); - /* Every 8 bits holds the sum of every 8-set of bits (5 significant bits) (8) */ - su_int x = (su_int)(x2 + (x2 >> 32)); - /* Every 8 bits holds the sum of every 8-set of bits (6 significant bits) (4) */ - x = x + (x >> 16); - /* Every 8 bits holds the sum of every 8-set of bits (7 significant bits) (2) */ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0xFF; /* (8 significant bits) */ +COMPILER_RT_ABI si_int __popcountti2(ti_int a) { + tu_int x3 = (tu_int)a; + x3 = x3 - ((x3 >> 1) & + (((tu_int)0x5555555555555555uLL << 64) | 0x5555555555555555uLL)); + // Every 2 bits holds the sum of every pair of bits (64) + x3 = ((x3 >> 2) & + (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)) + + (x3 & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (32) + x3 = (x3 + (x3 >> 4)) & + (((tu_int)0x0F0F0F0F0F0F0F0FuLL << 64) | 0x0F0F0F0F0F0F0F0FuLL); + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (16) + du_int x2 = (du_int)(x3 + (x3 >> 64)); + // Every 8 bits holds the sum of every 8-set of bits (5 significant bits) (8) + su_int x = (su_int)(x2 + (x2 >> 32)); + // Every 8 bits holds the sum of every 8-set of bits (6 significant bits) (4) + x = x + (x >> 16); + // Every 8 bits holds the sum of every 8-set of bits (7 significant bits) (2) + // + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0xFF; // (8 significant bits) } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/powidf2.c b/lib/builtins/powidf2.c index ac13b172b..969758848 100644 --- a/lib/builtins/powidf2.c +++ b/lib/builtins/powidf2.c @@ -1,34 +1,29 @@ -/* ===-- powidf2.cpp - Implement __powidf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powidf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powidf2.cpp - Implement __powidf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powidf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI double -__powidf2(double a, si_int b) -{ - const int recip = b < 0; - double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI double __powidf2(double a, si_int b) { + const int recip = b < 0; + double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } diff --git a/lib/builtins/powisf2.c b/lib/builtins/powisf2.c index 0c400ec6d..469402348 100644 --- a/lib/builtins/powisf2.c +++ b/lib/builtins/powisf2.c @@ -1,34 +1,29 @@ -/*===-- powisf2.cpp - Implement __powisf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powisf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powisf2.cpp - Implement __powisf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powisf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI float -__powisf2(float a, si_int b) -{ - const int recip = b < 0; - float r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI float __powisf2(float a, si_int b) { + const int recip = b < 0; + float r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } diff --git a/lib/builtins/powitf2.c b/lib/builtins/powitf2.c index 172f29f58..fcbdb4c2e 100644 --- a/lib/builtins/powitf2.c +++ b/lib/builtins/powitf2.c @@ -1,38 +1,33 @@ -/* ===-- powitf2.cpp - Implement __powitf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powitf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powitf2.cpp - Implement __powitf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powitf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #if _ARCH_PPC -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI long double -__powitf2(long double a, si_int b) -{ - const int recip = b < 0; - long double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI long double __powitf2(long double a, si_int b) { + const int recip = b < 0; + long double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } #endif diff --git a/lib/builtins/powixf2.c b/lib/builtins/powixf2.c index 0fd96e503..b7b52095a 100644 --- a/lib/builtins/powixf2.c +++ b/lib/builtins/powixf2.c @@ -1,38 +1,33 @@ -/* ===-- powixf2.cpp - Implement __powixf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powixf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powixf2.cpp - Implement __powixf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powixf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI long double -__powixf2(long double a, si_int b) -{ - const int recip = b < 0; - long double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI long double __powixf2(long double a, si_int b) { + const int recip = b < 0; + long double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } #endif diff --git a/lib/builtins/ppc/DD.h b/lib/builtins/ppc/DD.h index 3e5f9e58c..8f31a962f 100644 --- a/lib/builtins/ppc/DD.h +++ b/lib/builtins/ppc/DD.h @@ -4,20 +4,20 @@ #include "../int_lib.h" typedef union { - long double ld; - struct { - double hi; - double lo; - }s; + long double ld; + struct { + double hi; + double lo; + } s; } DD; -typedef union { - double d; - uint64_t x; +typedef union { + double d; + uint64_t x; } doublebits; -#define LOWORDER(xy,xHi,xLo,yHi,yLo) \ - (((((xHi)*(yHi) - (xy)) + (xHi)*(yLo)) + (xLo)*(yHi)) + (xLo)*(yLo)) +#define LOWORDER(xy, xHi, xLo, yHi, yLo) \ + (((((xHi) * (yHi) - (xy)) + (xHi) * (yLo)) + (xLo) * (yHi)) + (xLo) * (yLo)) static __inline ALWAYS_INLINE double local_fabs(double x) { doublebits result = {.d = x}; @@ -42,4 +42,4 @@ long double __gcc_qsub(long double, long double); long double __gcc_qmul(long double, long double); long double __gcc_qdiv(long double, long double); -#endif /* COMPILERRT_DD_HEADER */ +#endif // COMPILERRT_DD_HEADER diff --git a/lib/builtins/ppc/divtc3.c b/lib/builtins/ppc/divtc3.c index ef532b841..afaccf5a8 100644 --- a/lib/builtins/ppc/divtc3.c +++ b/lib/builtins/ppc/divtc3.c @@ -1,9 +1,9 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "DD.h" #include "../int_math.h" +#include "DD.h" // Use DOUBLE_PRECISION because the soft-fp method we use is logb (on the upper // half of the long doubles), even though this file defines complex division for // 128-bit floats. @@ -12,86 +12,85 @@ #if !defined(CRT_INFINITY) && defined(HUGE_VAL) #define CRT_INFINITY HUGE_VAL -#endif /* CRT_INFINITY */ +#endif // CRT_INFINITY + +#define makeFinite(x) \ + { \ + (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ + } + +long double _Complex __divtc3(long double a, long double b, long double c, + long double d) { + DD cDD = {.ld = c}; + DD dDD = {.ld = d}; + + int ilogbw = 0; + const double logbw = + __compiler_rt_logb(crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); + + if (crt_isfinite(logbw)) { + ilogbw = (int)logbw; + + cDD.s.hi = crt_scalbn(cDD.s.hi, -ilogbw); + cDD.s.lo = crt_scalbn(cDD.s.lo, -ilogbw); + dDD.s.hi = crt_scalbn(dDD.s.hi, -ilogbw); + dDD.s.lo = crt_scalbn(dDD.s.lo, -ilogbw); + } + + const long double denom = + __gcc_qadd(__gcc_qmul(cDD.ld, cDD.ld), __gcc_qmul(dDD.ld, dDD.ld)); + const long double realNumerator = + __gcc_qadd(__gcc_qmul(a, cDD.ld), __gcc_qmul(b, dDD.ld)); + const long double imagNumerator = + __gcc_qsub(__gcc_qmul(b, cDD.ld), __gcc_qmul(a, dDD.ld)); + + DD real = {.ld = __gcc_qdiv(realNumerator, denom)}; + DD imag = {.ld = __gcc_qdiv(imagNumerator, denom)}; + + real.s.hi = crt_scalbn(real.s.hi, -ilogbw); + real.s.lo = crt_scalbn(real.s.lo, -ilogbw); + imag.s.hi = crt_scalbn(imag.s.hi, -ilogbw); + imag.s.lo = crt_scalbn(imag.s.lo, -ilogbw); -#define makeFinite(x) { \ - (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ + if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) { + DD aDD = {.ld = a}; + DD bDD = {.ld = b}; + DD rDD = {.ld = denom}; + + if ((rDD.s.hi == 0.0) && (!crt_isnan(aDD.s.hi) || !crt_isnan(bDD.s.hi))) { + real.s.hi = crt_copysign(CRT_INFINITY, cDD.s.hi) * aDD.s.hi; + real.s.lo = 0.0; + imag.s.hi = crt_copysign(CRT_INFINITY, cDD.s.hi) * bDD.s.hi; + imag.s.lo = 0.0; + } + + else if ((crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) && + crt_isfinite(cDD.s.hi) && crt_isfinite(dDD.s.hi)) { + makeFinite(aDD); + makeFinite(bDD); + real.s.hi = CRT_INFINITY * (aDD.s.hi * cDD.s.hi + bDD.s.hi * dDD.s.hi); + real.s.lo = 0.0; + imag.s.hi = CRT_INFINITY * (bDD.s.hi * cDD.s.hi - aDD.s.hi * dDD.s.hi); + imag.s.lo = 0.0; + } + + else if ((crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) && + crt_isfinite(aDD.s.hi) && crt_isfinite(bDD.s.hi)) { + makeFinite(cDD); + makeFinite(dDD); + real.s.hi = + crt_copysign(0.0, (aDD.s.hi * cDD.s.hi + bDD.s.hi * dDD.s.hi)); + real.s.lo = 0.0; + imag.s.hi = + crt_copysign(0.0, (bDD.s.hi * cDD.s.hi - aDD.s.hi * dDD.s.hi)); + imag.s.lo = 0.0; + } } -long double _Complex -__divtc3(long double a, long double b, long double c, long double d) -{ - DD cDD = { .ld = c }; - DD dDD = { .ld = d }; - - int ilogbw = 0; - const double logbw = __compiler_rt_logb( - crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); - - if (crt_isfinite(logbw)) - { - ilogbw = (int)logbw; - - cDD.s.hi = crt_scalbn(cDD.s.hi, -ilogbw); - cDD.s.lo = crt_scalbn(cDD.s.lo, -ilogbw); - dDD.s.hi = crt_scalbn(dDD.s.hi, -ilogbw); - dDD.s.lo = crt_scalbn(dDD.s.lo, -ilogbw); - } - - const long double denom = __gcc_qadd(__gcc_qmul(cDD.ld, cDD.ld), __gcc_qmul(dDD.ld, dDD.ld)); - const long double realNumerator = __gcc_qadd(__gcc_qmul(a,cDD.ld), __gcc_qmul(b,dDD.ld)); - const long double imagNumerator = __gcc_qsub(__gcc_qmul(b,cDD.ld), __gcc_qmul(a,dDD.ld)); - - DD real = { .ld = __gcc_qdiv(realNumerator, denom) }; - DD imag = { .ld = __gcc_qdiv(imagNumerator, denom) }; - - real.s.hi = crt_scalbn(real.s.hi, -ilogbw); - real.s.lo = crt_scalbn(real.s.lo, -ilogbw); - imag.s.hi = crt_scalbn(imag.s.hi, -ilogbw); - imag.s.lo = crt_scalbn(imag.s.lo, -ilogbw); - - if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) - { - DD aDD = { .ld = a }; - DD bDD = { .ld = b }; - DD rDD = { .ld = denom }; - - if ((rDD.s.hi == 0.0) && (!crt_isnan(aDD.s.hi) || - !crt_isnan(bDD.s.hi))) - { - real.s.hi = crt_copysign(CRT_INFINITY,cDD.s.hi) * aDD.s.hi; - real.s.lo = 0.0; - imag.s.hi = crt_copysign(CRT_INFINITY,cDD.s.hi) * bDD.s.hi; - imag.s.lo = 0.0; - } - - else if ((crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) && - crt_isfinite(cDD.s.hi) && crt_isfinite(dDD.s.hi)) - { - makeFinite(aDD); - makeFinite(bDD); - real.s.hi = CRT_INFINITY * (aDD.s.hi*cDD.s.hi + bDD.s.hi*dDD.s.hi); - real.s.lo = 0.0; - imag.s.hi = CRT_INFINITY * (bDD.s.hi*cDD.s.hi - aDD.s.hi*dDD.s.hi); - imag.s.lo = 0.0; - } - - else if ((crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) && - crt_isfinite(aDD.s.hi) && crt_isfinite(bDD.s.hi)) - { - makeFinite(cDD); - makeFinite(dDD); - real.s.hi = crt_copysign(0.0,(aDD.s.hi*cDD.s.hi + bDD.s.hi*dDD.s.hi)); - real.s.lo = 0.0; - imag.s.hi = crt_copysign(0.0,(bDD.s.hi*cDD.s.hi - aDD.s.hi*dDD.s.hi)); - imag.s.lo = 0.0; - } - } - - long double _Complex z; - __real__ z = real.ld; - __imag__ z = imag.ld; - - return z; + long double _Complex z; + __real__ z = real.ld; + __imag__ z = imag.ld; + + return z; } diff --git a/lib/builtins/ppc/fixtfdi.c b/lib/builtins/ppc/fixtfdi.c index 2c7c0f8e2..a97aaf095 100644 --- a/lib/builtins/ppc/fixtfdi.c +++ b/lib/builtins/ppc/fixtfdi.c @@ -1,104 +1,98 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* int64_t __fixunstfdi(long double x); - * This file implements the PowerPC 128-bit double-double -> int64_t conversion - */ +// int64_t __fixunstfdi(long double x); +// This file implements the PowerPC 128-bit double-double -> int64_t conversion -#include "DD.h" #include "../int_math.h" +#include "DD.h" + +uint64_t __fixtfdi(long double input) { + const DD x = {.ld = input}; + const doublebits hibits = {.d = x.s.hi}; + + const uint32_t absHighWord = + (uint32_t)(hibits.x >> 32) & UINT32_C(0x7fffffff); + const uint32_t absHighWordMinusOne = absHighWord - UINT32_C(0x3ff00000); + + // If (1.0 - tiny) <= input < 0x1.0p63: + if (UINT32_C(0x03f00000) > absHighWordMinusOne) { + // Do an unsigned conversion of the absolute value, then restore the sign. + const int unbiasedHeadExponent = absHighWordMinusOne >> 20; + + int64_t result = hibits.x & INT64_C(0x000fffffffffffff); // mantissa(hi) + result |= INT64_C(0x0010000000000000); // matissa(hi) with implicit bit + result <<= 10; // mantissa(hi) with one zero preceding bit. + + const int64_t hiNegationMask = ((int64_t)(hibits.x)) >> 63; + + // If the tail is non-zero, we need to patch in the tail bits. + if (0.0 != x.s.lo) { + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // At this point we have the mantissa of |tail| + // We need to negate it if head and tail have different signs. + const int64_t loNegationMask = ((int64_t)(lobits.x)) >> 63; + const int64_t negationMask = loNegationMask ^ hiNegationMask; + tailMantissa = (tailMantissa ^ negationMask) - negationMask; + + // Now we have the mantissa of tail as a signed 2s-complement integer + + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + + // Shift the tail mantissa into the right position, accounting for the + // bias of 10 that we shifted the head mantissa by. + tailMantissa >>= + (unbiasedHeadExponent - (biasedTailExponent - (1023 - 10))); + + result += tailMantissa; + } + + result >>= (62 - unbiasedHeadExponent); + + // Restore the sign of the result and return + result = (result ^ hiNegationMask) - hiNegationMask; + return result; + } + + // Edge cases handled here: + + // |x| < 1, result is zero. + if (1.0 > crt_fabs(x.s.hi)) + return INT64_C(0); + + // x very close to INT64_MIN, care must be taken to see which side we are on. + if (x.s.hi == -0x1.0p63) { + + int64_t result = INT64_MIN; + + if (0.0 < x.s.lo) { + // If the tail is positive, the correct result is something other than + // INT64_MIN. we'll need to figure out what it is. + + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // Now we negate the tailMantissa + tailMantissa = (tailMantissa ^ INT64_C(-1)) + INT64_C(1); + + // And shift it by the appropriate amount + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + tailMantissa >>= 1075 - biasedTailExponent; + + result -= tailMantissa; + } + + return result; + } -uint64_t __fixtfdi(long double input) -{ - const DD x = { .ld = input }; - const doublebits hibits = { .d = x.s.hi }; - - const uint32_t absHighWord = (uint32_t)(hibits.x >> 32) & UINT32_C(0x7fffffff); - const uint32_t absHighWordMinusOne = absHighWord - UINT32_C(0x3ff00000); - - /* If (1.0 - tiny) <= input < 0x1.0p63: */ - if (UINT32_C(0x03f00000) > absHighWordMinusOne) - { - /* Do an unsigned conversion of the absolute value, then restore the sign. */ - const int unbiasedHeadExponent = absHighWordMinusOne >> 20; - - int64_t result = hibits.x & INT64_C(0x000fffffffffffff); /* mantissa(hi) */ - result |= INT64_C(0x0010000000000000); /* matissa(hi) with implicit bit */ - result <<= 10; /* mantissa(hi) with one zero preceding bit. */ - - const int64_t hiNegationMask = ((int64_t)(hibits.x)) >> 63; - - /* If the tail is non-zero, we need to patch in the tail bits. */ - if (0.0 != x.s.lo) - { - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* At this point we have the mantissa of |tail| */ - /* We need to negate it if head and tail have different signs. */ - const int64_t loNegationMask = ((int64_t)(lobits.x)) >> 63; - const int64_t negationMask = loNegationMask ^ hiNegationMask; - tailMantissa = (tailMantissa ^ negationMask) - negationMask; - - /* Now we have the mantissa of tail as a signed 2s-complement integer */ - - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - - /* Shift the tail mantissa into the right position, accounting for the - * bias of 10 that we shifted the head mantissa by. - */ - tailMantissa >>= (unbiasedHeadExponent - (biasedTailExponent - (1023 - 10))); - - result += tailMantissa; - } - - result >>= (62 - unbiasedHeadExponent); - - /* Restore the sign of the result and return */ - result = (result ^ hiNegationMask) - hiNegationMask; - return result; - - } - - /* Edge cases handled here: */ - - /* |x| < 1, result is zero. */ - if (1.0 > crt_fabs(x.s.hi)) - return INT64_C(0); - - /* x very close to INT64_MIN, care must be taken to see which side we are on. */ - if (x.s.hi == -0x1.0p63) { - - int64_t result = INT64_MIN; - - if (0.0 < x.s.lo) - { - /* If the tail is positive, the correct result is something other than INT64_MIN. - * we'll need to figure out what it is. - */ - - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* Now we negate the tailMantissa */ - tailMantissa = (tailMantissa ^ INT64_C(-1)) + INT64_C(1); - - /* And shift it by the appropriate amount */ - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - tailMantissa >>= 1075 - biasedTailExponent; - - result -= tailMantissa; - } - - return result; - } - - /* Signed overflows, infinities, and NaNs */ - if (x.s.hi > 0.0) - return INT64_MAX; - else - return INT64_MIN; + // Signed overflows, infinities, and NaNs + if (x.s.hi > 0.0) + return INT64_MAX; + else + return INT64_MIN; } diff --git a/lib/builtins/ppc/fixunstfdi.c b/lib/builtins/ppc/fixunstfdi.c index 5e6e2cedf..8d53f3725 100644 --- a/lib/builtins/ppc/fixunstfdi.c +++ b/lib/builtins/ppc/fixunstfdi.c @@ -1,59 +1,57 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* uint64_t __fixunstfdi(long double x); */ -/* This file implements the PowerPC 128-bit double-double -> uint64_t conversion */ +// uint64_t __fixunstfdi(long double x); +// This file implements the PowerPC 128-bit double-double -> uint64_t conversion #include "DD.h" -uint64_t __fixunstfdi(long double input) -{ - const DD x = { .ld = input }; - const doublebits hibits = { .d = x.s.hi }; - - const uint32_t highWordMinusOne = (uint32_t)(hibits.x >> 32) - UINT32_C(0x3ff00000); - - /* If (1.0 - tiny) <= input < 0x1.0p64: */ - if (UINT32_C(0x04000000) > highWordMinusOne) - { - const int unbiasedHeadExponent = highWordMinusOne >> 20; - - uint64_t result = hibits.x & UINT64_C(0x000fffffffffffff); /* mantissa(hi) */ - result |= UINT64_C(0x0010000000000000); /* matissa(hi) with implicit bit */ - result <<= 11; /* mantissa(hi) left aligned in the int64 field. */ - - /* If the tail is non-zero, we need to patch in the tail bits. */ - if (0.0 != x.s.lo) - { - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* At this point we have the mantissa of |tail| */ - - const int64_t negationMask = ((int64_t)(lobits.x)) >> 63; - tailMantissa = (tailMantissa ^ negationMask) - negationMask; - - /* Now we have the mantissa of tail as a signed 2s-complement integer */ - - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - - /* Shift the tail mantissa into the right position, accounting for the - * bias of 11 that we shifted the head mantissa by. - */ - tailMantissa >>= (unbiasedHeadExponent - (biasedTailExponent - (1023 - 11))); - - result += tailMantissa; - } - - result >>= (63 - unbiasedHeadExponent); - return result; - } - - /* Edge cases are handled here, with saturation. */ - if (1.0 > x.s.hi) - return UINT64_C(0); - else - return UINT64_MAX; +uint64_t __fixunstfdi(long double input) { + const DD x = {.ld = input}; + const doublebits hibits = {.d = x.s.hi}; + + const uint32_t highWordMinusOne = + (uint32_t)(hibits.x >> 32) - UINT32_C(0x3ff00000); + + // If (1.0 - tiny) <= input < 0x1.0p64: + if (UINT32_C(0x04000000) > highWordMinusOne) { + const int unbiasedHeadExponent = highWordMinusOne >> 20; + + uint64_t result = hibits.x & UINT64_C(0x000fffffffffffff); // mantissa(hi) + result |= UINT64_C(0x0010000000000000); // matissa(hi) with implicit bit + result <<= 11; // mantissa(hi) left aligned in the int64 field. + + // If the tail is non-zero, we need to patch in the tail bits. + if (0.0 != x.s.lo) { + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // At this point we have the mantissa of |tail| + + const int64_t negationMask = ((int64_t)(lobits.x)) >> 63; + tailMantissa = (tailMantissa ^ negationMask) - negationMask; + + // Now we have the mantissa of tail as a signed 2s-complement integer + + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + + // Shift the tail mantissa into the right position, accounting for the + // bias of 11 that we shifted the head mantissa by. + tailMantissa >>= + (unbiasedHeadExponent - (biasedTailExponent - (1023 - 11))); + + result += tailMantissa; + } + + result >>= (63 - unbiasedHeadExponent); + return result; + } + + // Edge cases are handled here, with saturation. + if (1.0 > x.s.hi) + return UINT64_C(0); + else + return UINT64_MAX; } diff --git a/lib/builtins/ppc/fixunstfti.c b/lib/builtins/ppc/fixunstfti.c index fa21084cb..1d19e01e3 100644 --- a/lib/builtins/ppc/fixunstfti.c +++ b/lib/builtins/ppc/fixunstfti.c @@ -1,9 +1,8 @@ //===-- lib/builtins/ppc/fixunstfti.c - Convert long double->int128 *-C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,10 +14,10 @@ #include "../int_math.h" #define BIAS 1023 -/* Convert long double into an unsigned 128-bit integer. */ +// Convert long double into an unsigned 128-bit integer. __uint128_t __fixunstfti(long double input) { - /* If we are trying to convert a NaN, return the NaN bit pattern. */ + // If we are trying to convert a NaN, return the NaN bit pattern. if (crt_isnan(input)) { return ((__uint128_t)0x7FF8000000000000ll) << 64 | (__uint128_t)0x0000000000000000ll; @@ -26,59 +25,59 @@ __uint128_t __fixunstfti(long double input) { __uint128_t result, hiResult, loResult; int hiExponent, loExponent, shift; - /* The long double representation, with the high and low portions of - * the long double, and the corresponding bit patterns of each double. */ + // The long double representation, with the high and low portions of + // the long double, and the corresponding bit patterns of each double. union { long double ld; - double d[2]; /* [0] is the high double, [1] is the low double. */ - unsigned long long ull[2]; /* High and low doubles as 64-bit integers. */ + double d[2]; // [0] is the high double, [1] is the low double. + unsigned long long ull[2]; // High and low doubles as 64-bit integers. } ldUnion; - /* If the long double is less than 1.0 or negative, - * return 0.0. */ + // If the long double is less than 1.0 or negative, + // return 0.0. if (input < 1.0) return 0.0; - /* Retrieve the 64-bit patterns of high and low doubles. - * Compute the unbiased exponent of both high and low doubles by - * removing the signs, isolating the exponent, and subtracting - * the bias from it. */ + // Retrieve the 64-bit patterns of high and low doubles. + // Compute the unbiased exponent of both high and low doubles by + // removing the signs, isolating the exponent, and subtracting + // the bias from it. ldUnion.ld = input; hiExponent = ((ldUnion.ull[0] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; loExponent = ((ldUnion.ull[1] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; - /* Convert each double into int64; they will be added to the int128 result. - * CASE 1: High or low double fits in int64 - * - Convert the each double normally into int64. - * - * CASE 2: High or low double does not fit in int64 - * - Scale the double to fit within a 64-bit integer - * - Calculate the shift (amount to scale the double by in the int128) - * - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) - * - Add BIAS+53 (0x4350000000000000) to exponent to correct the value - * - Scale (move) the double to the correct place in the int128 - * (Move it by 2^53 places) - * - * Note: If the high double is assumed to be positive, an unsigned conversion - * from long double to 64-bit integer is needed. The low double can be either - * positive or negative, so a signed conversion is needed to retain the result - * of the low double and to ensure it does not simply get converted to 0. */ + // Convert each double into int64; they will be added to the int128 result. + // CASE 1: High or low double fits in int64 + // - Convert the each double normally into int64. + // + // CASE 2: High or low double does not fit in int64 + // - Scale the double to fit within a 64-bit integer + // - Calculate the shift (amount to scale the double by in the int128) + // - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) + // - Add BIAS+53 (0x4350000000000000) to exponent to correct the value + // - Scale (move) the double to the correct place in the int128 + // (Move it by 2^53 places) + // + // Note: If the high double is assumed to be positive, an unsigned conversion + // from long double to 64-bit integer is needed. The low double can be either + // positive or negative, so a signed conversion is needed to retain the result + // of the low double and to ensure it does not simply get converted to 0. - /* CASE 1 - High double fits in int64. */ + // CASE 1 - High double fits in int64. if (hiExponent < 63) { hiResult = (unsigned long long)ldUnion.d[0]; } else if (hiExponent < 128) { - /* CASE 2 - High double does not fit in int64, scale and convert it. */ + // CASE 2 - High double does not fit in int64, scale and convert it. shift = hiExponent - 54; ldUnion.ull[0] &= 0x800FFFFFFFFFFFFFll; ldUnion.ull[0] |= 0x4350000000000000ll; hiResult = (unsigned long long)ldUnion.d[0]; hiResult <<= shift; } else { - /* Detect cases for overflow. When the exponent of the high - * double is greater than 128 bits and when the long double - * input is positive, return the max 128-bit integer. - * For negative inputs with exponents > 128, return 1, like gcc. */ + // Detect cases for overflow. When the exponent of the high + // double is greater than 128 bits and when the long double + // input is positive, return the max 128-bit integer. + // For negative inputs with exponents > 128, return 1, like gcc. if (ldUnion.d[0] > 0) { return ((__uint128_t)0xFFFFFFFFFFFFFFFFll) << 64 | (__uint128_t)0xFFFFFFFFFFFFFFFFll; @@ -88,11 +87,11 @@ __uint128_t __fixunstfti(long double input) { } } - /* CASE 1 - Low double fits in int64. */ + // CASE 1 - Low double fits in int64. if (loExponent < 63) { loResult = (long long)ldUnion.d[1]; } else { - /* CASE 2 - Low double does not fit in int64, scale and convert it. */ + // CASE 2 - Low double does not fit in int64, scale and convert it. shift = loExponent - 54; ldUnion.ull[1] &= 0x800FFFFFFFFFFFFFll; ldUnion.ull[1] |= 0x4350000000000000ll; @@ -100,7 +99,7 @@ __uint128_t __fixunstfti(long double input) { loResult <<= shift; } - /* Add the high and low doublewords together to form a 128 bit integer. */ + // Add the high and low doublewords together to form a 128 bit integer. result = loResult + hiResult; return result; } diff --git a/lib/builtins/ppc/floatditf.c b/lib/builtins/ppc/floatditf.c index beabdd017..4c365418f 100644 --- a/lib/builtins/ppc/floatditf.c +++ b/lib/builtins/ppc/floatditf.c @@ -1,36 +1,33 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatditf(long long x); */ -/* This file implements the PowerPC long long -> long double conversion */ +// long double __floatditf(long long x); +// This file implements the PowerPC long long -> long double conversion #include "DD.h" long double __floatditf(int64_t a) { - - static const double twop32 = 0x1.0p32; - static const double twop52 = 0x1.0p52; - - doublebits low = { .d = twop52 }; - low.x |= a & UINT64_C(0x00000000ffffffff); /* 0x1.0p52 + low 32 bits of a. */ - - const double high_addend = (double)((int32_t)(a >> 32))*twop32 - twop52; - - /* At this point, we have two double precision numbers - * high_addend and low.d, and we wish to return their sum - * as a canonicalized long double: - */ - - /* This implementation sets the inexact flag spuriously. - * This could be avoided, but at some substantial cost. - */ - - DD result; - - result.s.hi = high_addend + low.d; - result.s.lo = (high_addend - result.s.hi) + low.d; - - return result.ld; - + + static const double twop32 = 0x1.0p32; + static const double twop52 = 0x1.0p52; + + doublebits low = {.d = twop52}; + low.x |= a & UINT64_C(0x00000000ffffffff); // 0x1.0p52 + low 32 bits of a. + + const double high_addend = (double)((int32_t)(a >> 32)) * twop32 - twop52; + + // At this point, we have two double precision numbers + // high_addend and low.d, and we wish to return their sum + // as a canonicalized long double: + + // This implementation sets the inexact flag spuriously. + // This could be avoided, but at some substantial cost. + + DD result; + + result.s.hi = high_addend + low.d; + result.s.lo = (high_addend - result.s.hi) + low.d; + + return result.ld; } diff --git a/lib/builtins/ppc/floattitf.c b/lib/builtins/ppc/floattitf.c index b8e297b6b..6deac6498 100644 --- a/lib/builtins/ppc/floattitf.c +++ b/lib/builtins/ppc/floattitf.c @@ -1,9 +1,8 @@ //===-- lib/builtins/ppc/floattitf.c - Convert int128->long double -*-C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,35 +13,34 @@ #include <stdint.h> -/* Conversions from signed and unsigned 64-bit int to long double. */ +// Conversions from signed and unsigned 64-bit int to long double. long double __floatditf(int64_t); long double __floatunditf(uint64_t); -/* Convert a signed 128-bit integer to long double. - * This uses the following property: Let hi and lo be 64-bits each, - * and let signed_val_k() and unsigned_val_k() be the value of the - * argument interpreted as a signed or unsigned k-bit integer. Then, - * - * signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) - * = (long double)hi * 2^64 + (long double)lo, - * - * where (long double)hi and (long double)lo are signed and - * unsigned 64-bit integer to long double conversions, respectively. - */ +// Convert a signed 128-bit integer to long double. +// This uses the following property: Let hi and lo be 64-bits each, +// and let signed_val_k() and unsigned_val_k() be the value of the +// argument interpreted as a signed or unsigned k-bit integer. Then, +// +// signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) +// = (long double)hi * 2^64 + (long double)lo, +// +// where (long double)hi and (long double)lo are signed and +// unsigned 64-bit integer to long double conversions, respectively. long double __floattitf(__int128_t arg) { - /* Split the int128 argument into 64-bit high and low int64 parts. */ + // Split the int128 argument into 64-bit high and low int64 parts. int64_t ArgHiPart = (int64_t)(arg >> 64); uint64_t ArgLoPart = (uint64_t)arg; - /* Convert each 64-bit part into long double. The high part - * must be a signed conversion and the low part an unsigned conversion - * to ensure the correct result. */ + // Convert each 64-bit part into long double. The high part + // must be a signed conversion and the low part an unsigned conversion + // to ensure the correct result. long double ConvertedHiPart = __floatditf(ArgHiPart); long double ConvertedLoPart = __floatunditf(ArgLoPart); - /* The low bit of ArgHiPart corresponds to the 2^64 bit in arg. - * Multiply the high part by 2^64 to undo the right shift by 64-bits - * done in the splitting. Then, add to the low part to obtain the - * final result. */ + // The low bit of ArgHiPart corresponds to the 2^64 bit in arg. + // Multiply the high part by 2^64 to undo the right shift by 64-bits + // done in the splitting. Then, add to the low part to obtain the + // final result. return ((ConvertedHiPart * 0x1.0p64) + ConvertedLoPart); } diff --git a/lib/builtins/ppc/floatunditf.c b/lib/builtins/ppc/floatunditf.c index b12e1e738..fb4cd3f91 100644 --- a/lib/builtins/ppc/floatunditf.c +++ b/lib/builtins/ppc/floatunditf.c @@ -1,41 +1,39 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatunditf(unsigned long long x); */ -/* This file implements the PowerPC unsigned long long -> long double conversion */ +// long double __floatunditf(unsigned long long x); +// This file implements the PowerPC unsigned long long -> long double conversion #include "DD.h" long double __floatunditf(uint64_t a) { - - /* Begins with an exact copy of the code from __floatundidf */ - - static const double twop52 = 0x1.0p52; - static const double twop84 = 0x1.0p84; - static const double twop84_plus_twop52 = 0x1.00000001p84; - - doublebits high = { .d = twop84 }; - doublebits low = { .d = twop52 }; - - high.x |= a >> 32; /* 0x1.0p84 + high 32 bits of a */ - low.x |= a & UINT64_C(0x00000000ffffffff); /* 0x1.0p52 + low 32 bits of a */ - - const double high_addend = high.d - twop84_plus_twop52; - - /* At this point, we have two double precision numbers - * high_addend and low.d, and we wish to return their sum - * as a canonicalized long double: - */ - - /* This implementation sets the inexact flag spuriously. */ - /* This could be avoided, but at some substantial cost. */ - - DD result; - - result.s.hi = high_addend + low.d; - result.s.lo = (high_addend - result.s.hi) + low.d; - - return result.ld; - + + // Begins with an exact copy of the code from __floatundidf + + static const double twop52 = 0x1.0p52; + static const double twop84 = 0x1.0p84; + static const double twop84_plus_twop52 = 0x1.00000001p84; + + doublebits high = {.d = twop84}; + doublebits low = {.d = twop52}; + + high.x |= a >> 32; // 0x1.0p84 + high 32 bits of a + low.x |= a & UINT64_C(0x00000000ffffffff); // 0x1.0p52 + low 32 bits of a + + const double high_addend = high.d - twop84_plus_twop52; + + // At this point, we have two double precision numbers + // high_addend and low.d, and we wish to return their sum + // as a canonicalized long double: + + // This implementation sets the inexact flag spuriously. + // This could be avoided, but at some substantial cost. + + DD result; + + result.s.hi = high_addend + low.d; + result.s.lo = (high_addend - result.s.hi) + low.d; + + return result.ld; } diff --git a/lib/builtins/ppc/gcc_qadd.c b/lib/builtins/ppc/gcc_qadd.c index 32e16e9d1..6e1e63cb5 100644 --- a/lib/builtins/ppc/gcc_qadd.c +++ b/lib/builtins/ppc/gcc_qadd.c @@ -1,76 +1,74 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qadd(long double x, long double y); - * This file implements the PowerPC 128-bit double-double add operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qadd(long double x, long double y); +// This file implements the PowerPC 128-bit double-double add operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qadd(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = src.s.hi, b = src.s.lo; - - /* If both operands are zero: */ - if ((A == 0.0) && (B == 0.0)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If either operand is NaN or infinity: */ - const doublebits abits = { .d = A }; - const doublebits bbits = { .d = B }; - if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || - (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If the computation overflows: */ - /* This may be playing things a little bit fast and loose, but it will do for a start. */ - const double testForOverflow = A + (B + (a + b)); - const doublebits testbits = { .d = testForOverflow }; - if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = testForOverflow; - dst.s.lo = 0.0; - return dst.ld; - } - - double H, h; - double T, t; - double W, w; - double Y; - - H = B + (A - (A + B)); - T = b + (a - (a + b)); - h = A + (B - (A + B)); - t = a + (b - (a + b)); - - if (local_fabs(A) <= local_fabs(B)) - w = (a + b) + h; - else - w = (a + b) + H; - - W = (A + B) + w; - Y = (A + B) - W; - Y += w; - - if (local_fabs(a) <= local_fabs(b)) - w = t + Y; - else - w = T + Y; - - dst.s.hi = Y = W + w; - dst.s.lo = (W - Y) + w; - - return dst.ld; +long double __gcc_qadd(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = src.s.hi, b = src.s.lo; + + // If both operands are zero: + if ((A == 0.0) && (B == 0.0)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If either operand is NaN or infinity: + const doublebits abits = {.d = A}; + const doublebits bbits = {.d = B}; + if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || + (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If the computation overflows: + // This may be playing things a little bit fast and loose, but it will do for + // a start. + const double testForOverflow = A + (B + (a + b)); + const doublebits testbits = {.d = testForOverflow}; + if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = testForOverflow; + dst.s.lo = 0.0; + return dst.ld; + } + + double H, h; + double T, t; + double W, w; + double Y; + + H = B + (A - (A + B)); + T = b + (a - (a + b)); + h = A + (B - (A + B)); + t = a + (b - (a + b)); + + if (local_fabs(A) <= local_fabs(B)) + w = (a + b) + h; + else + w = (a + b) + H; + + W = (A + B) + w; + Y = (A + B) - W; + Y += w; + + if (local_fabs(a) <= local_fabs(b)) + w = t + Y; + else + w = T + Y; + + dst.s.hi = Y = W + w; + dst.s.lo = (W - Y) + w; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qdiv.c b/lib/builtins/ppc/gcc_qdiv.c index 70aa00b64..35a3cbc3d 100644 --- a/lib/builtins/ppc/gcc_qdiv.c +++ b/lib/builtins/ppc/gcc_qdiv.c @@ -1,55 +1,52 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qdiv(long double x, long double y); - * This file implements the PowerPC 128-bit double-double division operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qdiv(long double x, long double y); +// This file implements the PowerPC 128-bit double-double division operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qdiv(long double a, long double b) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - DD dst = { .ld = a }, src = { .ld = b }; - - register double x = dst.s.hi, x1 = dst.s.lo, - y = src.s.hi, y1 = src.s.lo; - - double yHi, yLo, qHi, qLo; - double yq, tmp, q; - - q = x / y; - - /* Detect special cases */ - if (q == 0.0) { - dst.s.hi = q; - dst.s.lo = 0.0; - return dst.ld; - } - - const doublebits qBits = { .d = q }; - if (((uint32_t)(qBits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = q; - dst.s.lo = 0.0; - return dst.ld; - } - - yHi = high26bits(y); - qHi = high26bits(q); - - yq = y * q; - yLo = y - yHi; - qLo = q - qHi; - - tmp = LOWORDER(yq, yHi, yLo, qHi, qLo); - tmp = (x - yq) - tmp; - tmp = ((tmp + x1) - y1 * q) / y; - x = q + tmp; - - dst.s.lo = (q - x) + tmp; - dst.s.hi = x; - +long double __gcc_qdiv(long double a, long double b) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + DD dst = {.ld = a}, src = {.ld = b}; + + register double x = dst.s.hi, x1 = dst.s.lo, y = src.s.hi, y1 = src.s.lo; + + double yHi, yLo, qHi, qLo; + double yq, tmp, q; + + q = x / y; + + // Detect special cases + if (q == 0.0) { + dst.s.hi = q; + dst.s.lo = 0.0; + return dst.ld; + } + + const doublebits qBits = {.d = q}; + if (((uint32_t)(qBits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = q; + dst.s.lo = 0.0; return dst.ld; + } + + yHi = high26bits(y); + qHi = high26bits(q); + + yq = y * q; + yLo = y - yHi; + qLo = q - qHi; + + tmp = LOWORDER(yq, yHi, yLo, qHi, qLo); + tmp = (x - yq) - tmp; + tmp = ((tmp + x1) - y1 * q) / y; + x = q + tmp; + + dst.s.lo = (q - x) + tmp; + dst.s.hi = x; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qmul.c b/lib/builtins/ppc/gcc_qmul.c index fb4c5164c..75f519aad 100644 --- a/lib/builtins/ppc/gcc_qmul.c +++ b/lib/builtins/ppc/gcc_qmul.c @@ -1,53 +1,50 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qmul(long double x, long double y); - * This file implements the PowerPC 128-bit double-double multiply operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qmul(long double x, long double y); +// This file implements the PowerPC 128-bit double-double multiply operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qmul(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = src.s.hi, b = src.s.lo; - - double aHi, aLo, bHi, bLo; - double ab, tmp, tau; - - ab = A * B; - - /* Detect special cases */ - if (ab == 0.0) { - dst.s.hi = ab; - dst.s.lo = 0.0; - return dst.ld; - } - - const doublebits abBits = { .d = ab }; - if (((uint32_t)(abBits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = ab; - dst.s.lo = 0.0; - return dst.ld; - } - - /* Generic cases handled here. */ - aHi = high26bits(A); - bHi = high26bits(B); - aLo = A - aHi; - bLo = B - bHi; - - tmp = LOWORDER(ab, aHi, aLo, bHi, bLo); - tmp += (A * b + a * B); - tau = ab + tmp; - - dst.s.lo = (ab - tau) + tmp; - dst.s.hi = tau; - +long double __gcc_qmul(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = src.s.hi, b = src.s.lo; + + double aHi, aLo, bHi, bLo; + double ab, tmp, tau; + + ab = A * B; + + // Detect special cases + if (ab == 0.0) { + dst.s.hi = ab; + dst.s.lo = 0.0; + return dst.ld; + } + + const doublebits abBits = {.d = ab}; + if (((uint32_t)(abBits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = ab; + dst.s.lo = 0.0; return dst.ld; + } + + // Generic cases handled here. + aHi = high26bits(A); + bHi = high26bits(B); + aLo = A - aHi; + bLo = B - bHi; + + tmp = LOWORDER(ab, aHi, aLo, bHi, bLo); + tmp += (A * b + a * B); + tau = ab + tmp; + + dst.s.lo = (ab - tau) + tmp; + dst.s.hi = tau; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qsub.c b/lib/builtins/ppc/gcc_qsub.c index c092e24db..ac08120be 100644 --- a/lib/builtins/ppc/gcc_qsub.c +++ b/lib/builtins/ppc/gcc_qsub.c @@ -1,76 +1,74 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qsub(long double x, long double y); - * This file implements the PowerPC 128-bit double-double add operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qsub(long double x, long double y); +// This file implements the PowerPC 128-bit double-double add operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qsub(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = -src.s.hi, b = -src.s.lo; - - /* If both operands are zero: */ - if ((A == 0.0) && (B == 0.0)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If either operand is NaN or infinity: */ - const doublebits abits = { .d = A }; - const doublebits bbits = { .d = B }; - if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || - (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If the computation overflows: */ - /* This may be playing things a little bit fast and loose, but it will do for a start. */ - const double testForOverflow = A + (B + (a + b)); - const doublebits testbits = { .d = testForOverflow }; - if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = testForOverflow; - dst.s.lo = 0.0; - return dst.ld; - } - - double H, h; - double T, t; - double W, w; - double Y; - - H = B + (A - (A + B)); - T = b + (a - (a + b)); - h = A + (B - (A + B)); - t = a + (b - (a + b)); - - if (local_fabs(A) <= local_fabs(B)) - w = (a + b) + h; - else - w = (a + b) + H; - - W = (A + B) + w; - Y = (A + B) - W; - Y += w; - - if (local_fabs(a) <= local_fabs(b)) - w = t + Y; - else - w = T + Y; - - dst.s.hi = Y = W + w; - dst.s.lo = (W - Y) + w; - - return dst.ld; +long double __gcc_qsub(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = -src.s.hi, b = -src.s.lo; + + // If both operands are zero: + if ((A == 0.0) && (B == 0.0)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If either operand is NaN or infinity: + const doublebits abits = {.d = A}; + const doublebits bbits = {.d = B}; + if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || + (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If the computation overflows: + // This may be playing things a little bit fast and loose, but it will do for + // a start. + const double testForOverflow = A + (B + (a + b)); + const doublebits testbits = {.d = testForOverflow}; + if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = testForOverflow; + dst.s.lo = 0.0; + return dst.ld; + } + + double H, h; + double T, t; + double W, w; + double Y; + + H = B + (A - (A + B)); + T = b + (a - (a + b)); + h = A + (B - (A + B)); + t = a + (b - (a + b)); + + if (local_fabs(A) <= local_fabs(B)) + w = (a + b) + h; + else + w = (a + b) + H; + + W = (A + B) + w; + Y = (A + B) - W; + Y += w; + + if (local_fabs(a) <= local_fabs(b)) + w = t + Y; + else + w = T + Y; + + dst.s.hi = Y = W + w; + dst.s.lo = (W - Y) + w; + + return dst.ld; } diff --git a/lib/builtins/ppc/multc3.c b/lib/builtins/ppc/multc3.c index 9dd79c975..f1fd6816d 100644 --- a/lib/builtins/ppc/multc3.c +++ b/lib/builtins/ppc/multc3.c @@ -1,90 +1,85 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "DD.h" #include "../int_math.h" +#include "DD.h" -#define makeFinite(x) { \ - (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ +#define makeFinite(x) \ + { \ + (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ } -#define zeroNaN(x) { \ - if (crt_isnan((x).s.hi)) { \ - (x).s.hi = crt_copysign(0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ - } \ +#define zeroNaN(x) \ + { \ + if (crt_isnan((x).s.hi)) { \ + (x).s.hi = crt_copysign(0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ + } \ } -long double _Complex -__multc3(long double a, long double b, long double c, long double d) -{ - long double ac = __gcc_qmul(a,c); - long double bd = __gcc_qmul(b,d); - long double ad = __gcc_qmul(a,d); - long double bc = __gcc_qmul(b,c); - - DD real = { .ld = __gcc_qsub(ac,bd) }; - DD imag = { .ld = __gcc_qadd(ad,bc) }; - - if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) - { - int recalc = 0; - - DD aDD = { .ld = a }; - DD bDD = { .ld = b }; - DD cDD = { .ld = c }; - DD dDD = { .ld = d }; - - if (crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) - { - makeFinite(aDD); - makeFinite(bDD); - zeroNaN(cDD); - zeroNaN(dDD); - recalc = 1; - } - - if (crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) - { - makeFinite(cDD); - makeFinite(dDD); - zeroNaN(aDD); - zeroNaN(bDD); - recalc = 1; - } - - if (!recalc) - { - DD acDD = { .ld = ac }; - DD bdDD = { .ld = bd }; - DD adDD = { .ld = ad }; - DD bcDD = { .ld = bc }; - - if (crt_isinf(acDD.s.hi) || crt_isinf(bdDD.s.hi) || - crt_isinf(adDD.s.hi) || crt_isinf(bcDD.s.hi)) - { - zeroNaN(aDD); - zeroNaN(bDD); - zeroNaN(cDD); - zeroNaN(dDD); - recalc = 1; - } - } - - if (recalc) - { - real.s.hi = CRT_INFINITY * (aDD.s.hi*cDD.s.hi - bDD.s.hi*dDD.s.hi); - real.s.lo = 0.0; - imag.s.hi = CRT_INFINITY * (aDD.s.hi*dDD.s.hi + bDD.s.hi*cDD.s.hi); - imag.s.lo = 0.0; - } - } - - long double _Complex z; - __real__ z = real.ld; - __imag__ z = imag.ld; - - return z; +long double _Complex __multc3(long double a, long double b, long double c, + long double d) { + long double ac = __gcc_qmul(a, c); + long double bd = __gcc_qmul(b, d); + long double ad = __gcc_qmul(a, d); + long double bc = __gcc_qmul(b, c); + + DD real = {.ld = __gcc_qsub(ac, bd)}; + DD imag = {.ld = __gcc_qadd(ad, bc)}; + + if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) { + int recalc = 0; + + DD aDD = {.ld = a}; + DD bDD = {.ld = b}; + DD cDD = {.ld = c}; + DD dDD = {.ld = d}; + + if (crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) { + makeFinite(aDD); + makeFinite(bDD); + zeroNaN(cDD); + zeroNaN(dDD); + recalc = 1; + } + + if (crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) { + makeFinite(cDD); + makeFinite(dDD); + zeroNaN(aDD); + zeroNaN(bDD); + recalc = 1; + } + + if (!recalc) { + DD acDD = {.ld = ac}; + DD bdDD = {.ld = bd}; + DD adDD = {.ld = ad}; + DD bcDD = {.ld = bc}; + + if (crt_isinf(acDD.s.hi) || crt_isinf(bdDD.s.hi) || + crt_isinf(adDD.s.hi) || crt_isinf(bcDD.s.hi)) { + zeroNaN(aDD); + zeroNaN(bDD); + zeroNaN(cDD); + zeroNaN(dDD); + recalc = 1; + } + } + + if (recalc) { + real.s.hi = CRT_INFINITY * (aDD.s.hi * cDD.s.hi - bDD.s.hi * dDD.s.hi); + real.s.lo = 0.0; + imag.s.hi = CRT_INFINITY * (aDD.s.hi * dDD.s.hi + bDD.s.hi * cDD.s.hi); + imag.s.lo = 0.0; + } + } + + long double _Complex z; + __real__ z = real.ld; + __imag__ z = imag.ld; + + return z; } diff --git a/lib/builtins/ppc/restFP.S b/lib/builtins/ppc/restFP.S index 507e756e1..02317bd6a 100644 --- a/lib/builtins/ppc/restFP.S +++ b/lib/builtins/ppc/restFP.S @@ -1,9 +1,8 @@ //===-- restFP.S - Implement restFP ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/ppc/saveFP.S b/lib/builtins/ppc/saveFP.S index 20b06fff5..1ef5532c8 100644 --- a/lib/builtins/ppc/saveFP.S +++ b/lib/builtins/ppc/saveFP.S @@ -1,9 +1,8 @@ //===-- saveFP.S - Implement saveFP ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/riscv/mulsi3.S b/lib/builtins/riscv/mulsi3.S index a58d23704..5464919b2 100644 --- a/lib/builtins/riscv/mulsi3.S +++ b/lib/builtins/riscv/mulsi3.S @@ -1,9 +1,8 @@ //===--- mulsi3.S - Integer multiplication routines routines ---===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/subdf3.c b/lib/builtins/subdf3.c index a892fa603..5346dbc97 100644 --- a/lib/builtins/subdf3.c +++ b/lib/builtins/subdf3.c @@ -1,9 +1,8 @@ //===-- lib/adddf3.c - Double-precision subtraction ---------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,17 +15,14 @@ #include "fp_lib.h" // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subdf3(fp_t a, fp_t b) { - return __adddf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subdf3(fp_t a, fp_t b) { + return __adddf3(a, fromRep(toRep(b) ^ signBit)); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { - return __subdf3(a, b); -} +AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { return __subdf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subdf3); +COMPILER_RT_ALIAS(__subdf3, __aeabi_dsub) #endif #endif diff --git a/lib/builtins/subsf3.c b/lib/builtins/subsf3.c index 4b2786177..85bde029b 100644 --- a/lib/builtins/subsf3.c +++ b/lib/builtins/subsf3.c @@ -1,9 +1,8 @@ //===-- lib/subsf3.c - Single-precision subtraction ---------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,17 +15,14 @@ #include "fp_lib.h" // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subsf3(fp_t a, fp_t b) { - return __addsf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subsf3(fp_t a, fp_t b) { + return __addsf3(a, fromRep(toRep(b) ^ signBit)); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { - return __subsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { return __subsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subsf3); +COMPILER_RT_ALIAS(__subsf3, __aeabi_fsub) #endif #endif diff --git a/lib/builtins/subtf3.c b/lib/builtins/subtf3.c index 609b816f4..c96814692 100644 --- a/lib/builtins/subtf3.c +++ b/lib/builtins/subtf3.c @@ -1,9 +1,8 @@ //===-- lib/subtf3.c - Quad-precision subtraction -----------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,9 +18,8 @@ COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b); // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subtf3(fp_t a, fp_t b) { - return __addtf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subtf3(fp_t a, fp_t b) { + return __addtf3(a, fromRep(toRep(b) ^ signBit)); } #endif diff --git a/lib/builtins/subvdi3.c b/lib/builtins/subvdi3.c index 71fc70ffa..d7d78f1bb 100644 --- a/lib/builtins/subvdi3.c +++ b/lib/builtins/subvdi3.c @@ -1,36 +1,29 @@ -/* ===-- subvdi3.c - Implement __subvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvdi3.c - Implement __subvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI di_int -__subvdi3(di_int a, di_int b) -{ - di_int s = (du_int) a - (du_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI di_int __subvdi3(di_int a, di_int b) { + di_int s = (du_int)a - (du_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/subvsi3.c b/lib/builtins/subvsi3.c index e6c0fb688..c3cb6e8a2 100644 --- a/lib/builtins/subvsi3.c +++ b/lib/builtins/subvsi3.c @@ -1,36 +1,29 @@ -/* ===-- subvsi3.c - Implement __subvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvsi3.c - Implement __subvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI si_int -__subvsi3(si_int a, si_int b) -{ - si_int s = (su_int) a - (su_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI si_int __subvsi3(si_int a, si_int b) { + si_int s = (su_int)a - (su_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/subvti3.c b/lib/builtins/subvti3.c index a6804d2d7..91ac18834 100644 --- a/lib/builtins/subvti3.c +++ b/lib/builtins/subvti3.c @@ -1,40 +1,33 @@ -/* ===-- subvti3.c - Implement __subvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvti3.c - Implement __subvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI ti_int -__subvti3(ti_int a, ti_int b) -{ - ti_int s = (tu_int) a - (tu_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI ti_int __subvti3(ti_int a, ti_int b) { + ti_int s = (tu_int)a - (tu_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/trampoline_setup.c b/lib/builtins/trampoline_setup.c index 25b627ab7..a62431723 100644 --- a/lib/builtins/trampoline_setup.c +++ b/lib/builtins/trampoline_setup.c @@ -1,48 +1,43 @@ -/* ===----- trampoline_setup.c - Implement __trampoline_setup -------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===----- trampoline_setup.c - Implement __trampoline_setup -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -extern void __clear_cache(void* start, void* end); +extern void __clear_cache(void *start, void *end); -/* - * The ppc compiler generates calls to __trampoline_setup() when creating - * trampoline functions on the stack for use with nested functions. - * This function creates a custom 40-byte trampoline function on the stack - * which loads r11 with a pointer to the outer function's locals - * and then jumps to the target nested function. - */ +// The ppc compiler generates calls to __trampoline_setup() when creating +// trampoline functions on the stack for use with nested functions. +// This function creates a custom 40-byte trampoline function on the stack +// which loads r11 with a pointer to the outer function's locals +// and then jumps to the target nested function. #if __ppc__ && !defined(__powerpc64__) -COMPILER_RT_ABI void -__trampoline_setup(uint32_t* trampOnStack, int trampSizeAllocated, - const void* realFunc, void* localsPtr) -{ - /* should never happen, but if compiler did not allocate */ - /* enough space on stack for the trampoline, abort */ - if ( trampSizeAllocated < 40 ) - compilerrt_abort(); - - /* create trampoline */ - trampOnStack[0] = 0x7c0802a6; /* mflr r0 */ - trampOnStack[1] = 0x4800000d; /* bl Lbase */ - trampOnStack[2] = (uint32_t)realFunc; - trampOnStack[3] = (uint32_t)localsPtr; - trampOnStack[4] = 0x7d6802a6; /* Lbase: mflr r11 */ - trampOnStack[5] = 0x818b0000; /* lwz r12,0(r11) */ - trampOnStack[6] = 0x7c0803a6; /* mtlr r0 */ - trampOnStack[7] = 0x7d8903a6; /* mtctr r12 */ - trampOnStack[8] = 0x816b0004; /* lwz r11,4(r11) */ - trampOnStack[9] = 0x4e800420; /* bctr */ - - /* clear instruction cache */ - __clear_cache(trampOnStack, &trampOnStack[10]); +COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, + int trampSizeAllocated, + const void *realFunc, void *localsPtr) { + // should never happen, but if compiler did not allocate + // enough space on stack for the trampoline, abort + if (trampSizeAllocated < 40) + compilerrt_abort(); + + // create trampoline + trampOnStack[0] = 0x7c0802a6; // mflr r0 + trampOnStack[1] = 0x4800000d; // bl Lbase + trampOnStack[2] = (uint32_t)realFunc; + trampOnStack[3] = (uint32_t)localsPtr; + trampOnStack[4] = 0x7d6802a6; // Lbase: mflr r11 + trampOnStack[5] = 0x818b0000; // lwz r12,0(r11) + trampOnStack[6] = 0x7c0803a6; // mtlr r0 + trampOnStack[7] = 0x7d8903a6; // mtctr r12 + trampOnStack[8] = 0x816b0004; // lwz r11,4(r11) + trampOnStack[9] = 0x4e800420; // bctr + + // clear instruction cache + __clear_cache(trampOnStack, &trampOnStack[10]); } -#endif /* __ppc__ && !defined(__powerpc64__) */ +#endif // __ppc__ && !defined(__powerpc64__) diff --git a/lib/builtins/truncdfhf2.c b/lib/builtins/truncdfhf2.c index 8354a41b8..90c418a43 100644 --- a/lib/builtins/truncdfhf2.c +++ b/lib/builtins/truncdfhf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfhf2.c - double -> half conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,16 +10,12 @@ #define DST_HALF #include "fp_trunc_impl.inc" -COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI uint16_t __aeabi_d2h(double a) { - return __truncdfhf2(a); -} +AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); } #else -AEABI_RTABI uint16_t __aeabi_d2h(double a) COMPILER_RT_ALIAS(__truncdfhf2); +COMPILER_RT_ALIAS(__truncdfhf2, __aeabi_d2h) #endif #endif diff --git a/lib/builtins/truncdfsf2.c b/lib/builtins/truncdfsf2.c index 195d3e065..44a1299e3 100644 --- a/lib/builtins/truncdfsf2.c +++ b/lib/builtins/truncdfsf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfsf2.c - double -> single conversion ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,16 +10,12 @@ #define DST_SINGLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI float __truncdfsf2(double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI float __truncdfsf2(double a) { return __truncXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_d2f(double a) { - return __truncdfsf2(a); -} +AEABI_RTABI float __aeabi_d2f(double a) { return __truncdfsf2(a); } #else -AEABI_RTABI float __aeabi_d2f(double a) COMPILER_RT_ALIAS(__truncdfsf2); +COMPILER_RT_ALIAS(__truncdfsf2, __aeabi_d2f) #endif #endif diff --git a/lib/builtins/truncsfhf2.c b/lib/builtins/truncsfhf2.c index 9c84ab4f9..1f17194c3 100644 --- a/lib/builtins/truncsfhf2.c +++ b/lib/builtins/truncsfhf2.c @@ -1,9 +1,8 @@ //===-- lib/truncsfhf2.c - single -> half conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,19 +13,15 @@ // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) { - return __truncXfYf2__(a); + return __truncXfYf2__(a); } -COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { - return __truncsfhf2(a); -} +COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI uint16_t __aeabi_f2h(float a) { - return __truncsfhf2(a); -} +AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); } #else -AEABI_RTABI uint16_t __aeabi_f2h(float a) COMPILER_RT_ALIAS(__truncsfhf2); +COMPILER_RT_ALIAS(__truncsfhf2, __aeabi_f2h) #endif #endif diff --git a/lib/builtins/trunctfdf2.c b/lib/builtins/trunctfdf2.c index 741a71b33..6857ea54d 100644 --- a/lib/builtins/trunctfdf2.c +++ b/lib/builtins/trunctfdf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfsf2.c - quad -> double conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,8 +14,6 @@ #define DST_DOUBLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI double __trunctfdf2(long double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI double __trunctfdf2(long double a) { return __truncXfYf2__(a); } #endif diff --git a/lib/builtins/trunctfsf2.c b/lib/builtins/trunctfsf2.c index de96c1dec..0261b1e90 100644 --- a/lib/builtins/trunctfsf2.c +++ b/lib/builtins/trunctfsf2.c @@ -1,9 +1,8 @@ //===-- lib/trunctfsf2.c - quad -> single conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,8 +14,6 @@ #define DST_SINGLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI float __trunctfsf2(long double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI float __trunctfsf2(long double a) { return __truncXfYf2__(a); } #endif diff --git a/lib/builtins/ucmpdi2.c b/lib/builtins/ucmpdi2.c index 40af23613..0f2edde83 100644 --- a/lib/builtins/ucmpdi2.c +++ b/lib/builtins/ucmpdi2.c @@ -1,51 +1,42 @@ -/* ===-- ucmpdi2.c - Implement __ucmpdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ucmpdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ucmpdi2.c - Implement __ucmpdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ucmpdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__ucmpdi2(du_int a, du_int b) -{ - udwords x; - x.all = a; - udwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __ucmpdi2(du_int a, du_int b) { + udwords x; + x.all = a; + udwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } #ifdef __ARM_EABI__ -/* Returns: if (a < b) returns -1 -* if (a == b) returns 0 -* if (a > b) returns 1 -*/ -COMPILER_RT_ABI si_int -__aeabi_ulcmp(di_int a, di_int b) -{ - return __ucmpdi2(a, b) - 1; +// Returns: if (a < b) returns -1 +// if (a == b) returns 0 +// if (a > b) returns 1 +COMPILER_RT_ABI si_int __aeabi_ulcmp(di_int a, di_int b) { + return __ucmpdi2(a, b) - 1; } #endif - diff --git a/lib/builtins/ucmpti2.c b/lib/builtins/ucmpti2.c index bda8083bb..4eb6655b0 100644 --- a/lib/builtins/ucmpti2.c +++ b/lib/builtins/ucmpti2.c @@ -1,42 +1,37 @@ -/* ===-- ucmpti2.c - Implement __ucmpti2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ucmpti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ucmpti2.c - Implement __ucmpti2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ucmpti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__ucmpti2(tu_int a, tu_int b) -{ - utwords x; - x.all = a; - utwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __ucmpti2(tu_int a, tu_int b) { + utwords x; + x.all = a; + utwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/udivdi3.c b/lib/builtins/udivdi3.c index dc68e154b..a23139ec9 100644 --- a/lib/builtins/udivdi3.c +++ b/lib/builtins/udivdi3.c @@ -1,23 +1,19 @@ -/* ===-- udivdi3.c - Implement __udivdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivdi3.c - Implement __udivdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI du_int -__udivdi3(du_int a, du_int b) -{ - return __udivmoddi4(a, b, 0); +COMPILER_RT_ABI du_int __udivdi3(du_int a, du_int b) { + return __udivmoddi4(a, b, 0); } diff --git a/lib/builtins/udivmoddi4.c b/lib/builtins/udivmoddi4.c index 0c8b4ff46..2914cc0fb 100644 --- a/lib/builtins/udivmoddi4.c +++ b/lib/builtins/udivmoddi4.c @@ -1,231 +1,189 @@ -/* ===-- udivmoddi4.c - Implement __udivmoddi4 -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmoddi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmoddi4.c - Implement __udivmoddi4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Effects: if rem != 0, *rem = a % b - * Returns: a / b - */ +// Effects: if rem != 0, *rem = a % b +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -COMPILER_RT_ABI du_int -__udivmoddi4(du_int a, du_int b, du_int* rem) -{ - const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; - const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; - udwords n; - n.all = a; - udwords d; - d.all = b; - udwords q; - udwords r; - unsigned sr; - /* special cases, X is unknown, K != 0 */ - if (n.s.high == 0) - { - if (d.s.high == 0) - { - /* 0 X - * --- - * 0 X - */ - if (rem) - *rem = n.s.low % d.s.low; - return n.s.low / d.s.low; - } - /* 0 X - * --- - * K X - */ - if (rem) - *rem = n.s.low; - return 0; +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) { + const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; + const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; + udwords n; + n.all = a; + udwords d; + d.all = b; + udwords q; + udwords r; + unsigned sr; + // special cases, X is unknown, K != 0 + if (n.s.high == 0) { + if (d.s.high == 0) { + // 0 X + // --- + // 0 X + if (rem) + *rem = n.s.low % d.s.low; + return n.s.low / d.s.low; + } + // 0 X + // --- + // K X + if (rem) + *rem = n.s.low; + return 0; + } + // n.s.high != 0 + if (d.s.low == 0) { + if (d.s.high == 0) { + // K X + // --- + // 0 0 + if (rem) + *rem = n.s.high % d.s.low; + return n.s.high / d.s.low; + } + // d.s.high != 0 + if (n.s.low == 0) { + // K 0 + // --- + // K 0 + if (rem) { + r.s.high = n.s.high % d.s.high; + r.s.low = 0; + *rem = r.all; + } + return n.s.high / d.s.high; + } + // K K + // --- + // K 0 + if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ { + if (rem) { + r.s.low = n.s.low; + r.s.high = n.s.high & (d.s.high - 1); + *rem = r.all; + } + return n.s.high >> __builtin_ctz(d.s.high); } - /* n.s.high != 0 */ - if (d.s.low == 0) - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 0 - */ - if (rem) - *rem = n.s.high % d.s.low; - return n.s.high / d.s.low; - } - /* d.s.high != 0 */ - if (n.s.low == 0) - { - /* K 0 - * --- - * K 0 - */ - if (rem) - { - r.s.high = n.s.high % d.s.high; - r.s.low = 0; - *rem = r.all; - } - return n.s.high / d.s.high; - } - /* K K - * --- - * K 0 - */ - if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - { - r.s.low = n.s.low; - r.s.high = n.s.high & (d.s.high - 1); - *rem = r.all; - } - return n.s.high >> __builtin_ctz(d.s.high); - } - /* K K - * --- - * K 0 - */ - sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); - /* 0 <= sr <= n_uword_bits - 2 or sr large */ - if (sr > n_uword_bits - 2) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_uword_bits - 1 */ - /* q.all = n.all << (n_udword_bits - sr); */ + // K K + // --- + // K 0 + sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); + // 0 <= sr <= n_uword_bits - 2 or sr large + if (sr > n_uword_bits - 2) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + q.s.high = n.s.low << (n_uword_bits - sr); + // r.all = n.all >> sr; + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* d.s.low != 0 */ { + if (d.s.high == 0) { + // K X + // --- + // 0 K + if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ { + if (rem) + *rem = n.s.low & (d.s.low - 1); + if (d.s.low == 1) + return n.all; + sr = __builtin_ctz(d.s.low); + q.s.high = n.s.high >> sr; + q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + return q.all; + } + // K X + // --- + // 0 K + sr = 1 + n_uword_bits + __builtin_clz(d.s.low) - __builtin_clz(n.s.high); + // 2 <= sr <= n_udword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + if (sr == n_uword_bits) { + q.s.low = 0; + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else if (sr < n_uword_bits) /* 2 <= sr <= n_uword_bits - 1 */ { q.s.low = 0; q.s.high = n.s.low << (n_uword_bits - sr); - /* r.all = n.all >> sr; */ r.s.high = n.s.high >> sr; r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* n_uword_bits + 1 <= sr <= n_udword_bits - 1 */ { + q.s.low = n.s.low << (n_udword_bits - sr); + q.s.high = (n.s.high << (n_udword_bits - sr)) | + (n.s.low >> (sr - n_uword_bits)); + r.s.high = 0; + r.s.low = n.s.high >> (sr - n_uword_bits); + } + } else { + // K X + // --- + // K K + sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); + // 0 <= sr <= n_uword_bits - 1 or sr large + if (sr > n_uword_bits - 1) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + if (sr == n_uword_bits) { + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else { + q.s.high = n.s.low << (n_uword_bits - sr); + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } } - else /* d.s.low != 0 */ - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 K - */ - if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - *rem = n.s.low & (d.s.low - 1); - if (d.s.low == 1) - return n.all; - sr = __builtin_ctz(d.s.low); - q.s.high = n.s.high >> sr; - q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - return q.all; - } - /* K X - * --- - * 0 K - */ - sr = 1 + n_uword_bits + __builtin_clz(d.s.low) - __builtin_clz(n.s.high); - /* 2 <= sr <= n_udword_bits - 1 - * q.all = n.all << (n_udword_bits - sr); - * r.all = n.all >> sr; - */ - if (sr == n_uword_bits) - { - q.s.low = 0; - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else if (sr < n_uword_bits) // 2 <= sr <= n_uword_bits - 1 - { - q.s.low = 0; - q.s.high = n.s.low << (n_uword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - } - else // n_uword_bits + 1 <= sr <= n_udword_bits - 1 - { - q.s.low = n.s.low << (n_udword_bits - sr); - q.s.high = (n.s.high << (n_udword_bits - sr)) | - (n.s.low >> (sr - n_uword_bits)); - r.s.high = 0; - r.s.low = n.s.high >> (sr - n_uword_bits); - } - } - else - { - /* K X - * --- - * K K - */ - sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); - /* 0 <= sr <= n_uword_bits - 1 or sr large */ - if (sr > n_uword_bits - 1) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_uword_bits */ - /* q.all = n.all << (n_udword_bits - sr); */ - q.s.low = 0; - if (sr == n_uword_bits) - { - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else - { - q.s.high = n.s.low << (n_uword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - } - } - } - /* Not a special case - * q and r are initialized with: - * q.all = n.all << (n_udword_bits - sr); - * r.all = n.all >> sr; - * 1 <= sr <= n_udword_bits - 1 - */ - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r.s.high = (r.s.high << 1) | (r.s.low >> (n_uword_bits - 1)); - r.s.low = (r.s.low << 1) | (q.s.high >> (n_uword_bits - 1)); - q.s.high = (q.s.high << 1) | (q.s.low >> (n_uword_bits - 1)); - q.s.low = (q.s.low << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const di_int s = (di_int)(d.all - r.all - 1) >> (n_udword_bits - 1); - carry = s & 1; - r.all -= d.all & s; - } - q.all = (q.all << 1) | carry; - if (rem) - *rem = r.all; - return q.all; + } + // Not a special case + // q and r are initialized with: + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + // 1 <= sr <= n_udword_bits - 1 + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r.s.high = (r.s.high << 1) | (r.s.low >> (n_uword_bits - 1)); + r.s.low = (r.s.low << 1) | (q.s.high >> (n_uword_bits - 1)); + q.s.high = (q.s.high << 1) | (q.s.low >> (n_uword_bits - 1)); + q.s.low = (q.s.low << 1) | carry; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const di_int s = (di_int)(d.all - r.all - 1) >> (n_udword_bits - 1); + carry = s & 1; + r.all -= d.all & s; + } + q.all = (q.all << 1) | carry; + if (rem) + *rem = r.all; + return q.all; } diff --git a/lib/builtins/udivmodsi4.c b/lib/builtins/udivmodsi4.c index 789c4b506..753ad6dd9 100644 --- a/lib/builtins/udivmodsi4.c +++ b/lib/builtins/udivmodsi4.c @@ -1,27 +1,21 @@ -/*===-- udivmodsi4.c - Implement __udivmodsi4 ------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmodsi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmodsi4.c - Implement __udivmodsi4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmodsi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI su_int -__udivmodsi4(su_int a, su_int b, su_int* rem) -{ - si_int d = __udivsi3(a,b); - *rem = a - (d*b); +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem) { + si_int d = __udivsi3(a, b); + *rem = a - (d * b); return d; } - - diff --git a/lib/builtins/udivmodti4.c b/lib/builtins/udivmodti4.c index 803168849..dd14a8b57 100644 --- a/lib/builtins/udivmodti4.c +++ b/lib/builtins/udivmodti4.c @@ -1,238 +1,195 @@ -/* ===-- udivmodti4.c - Implement __udivmodti4 -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmodti4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmodti4.c - Implement __udivmodti4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmodti4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Effects: if rem != 0, *rem = a % b - * Returns: a / b - */ +// Effects: if rem != 0, *rem = a % b +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -COMPILER_RT_ABI tu_int -__udivmodti4(tu_int a, tu_int b, tu_int* rem) -{ - const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; - const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT; - utwords n; - n.all = a; - utwords d; - d.all = b; - utwords q; - utwords r; - unsigned sr; - /* special cases, X is unknown, K != 0 */ - if (n.s.high == 0) - { - if (d.s.high == 0) - { - /* 0 X - * --- - * 0 X - */ - if (rem) - *rem = n.s.low % d.s.low; - return n.s.low / d.s.low; - } - /* 0 X - * --- - * K X - */ - if (rem) - *rem = n.s.low; - return 0; +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem) { + const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; + const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT; + utwords n; + n.all = a; + utwords d; + d.all = b; + utwords q; + utwords r; + unsigned sr; + // special cases, X is unknown, K != 0 + if (n.s.high == 0) { + if (d.s.high == 0) { + // 0 X + // --- + // 0 X + if (rem) + *rem = n.s.low % d.s.low; + return n.s.low / d.s.low; + } + // 0 X + // --- + // K X + if (rem) + *rem = n.s.low; + return 0; + } + // n.s.high != 0 + if (d.s.low == 0) { + if (d.s.high == 0) { + // K X + // --- + // 0 0 + if (rem) + *rem = n.s.high % d.s.low; + return n.s.high / d.s.low; + } + // d.s.high != 0 + if (n.s.low == 0) { + // K 0 + // --- + // K 0 + if (rem) { + r.s.high = n.s.high % d.s.high; + r.s.low = 0; + *rem = r.all; + } + return n.s.high / d.s.high; + } + // K K + // --- + // K 0 + if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ { + if (rem) { + r.s.low = n.s.low; + r.s.high = n.s.high & (d.s.high - 1); + *rem = r.all; + } + return n.s.high >> __builtin_ctzll(d.s.high); } - /* n.s.high != 0 */ - if (d.s.low == 0) - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 0 - */ - if (rem) - *rem = n.s.high % d.s.low; - return n.s.high / d.s.low; - } - /* d.s.high != 0 */ - if (n.s.low == 0) - { - /* K 0 - * --- - * K 0 - */ - if (rem) - { - r.s.high = n.s.high % d.s.high; - r.s.low = 0; - *rem = r.all; - } - return n.s.high / d.s.high; - } - /* K K - * --- - * K 0 - */ - if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - { - r.s.low = n.s.low; - r.s.high = n.s.high & (d.s.high - 1); - *rem = r.all; - } - return n.s.high >> __builtin_ctzll(d.s.high); - } - /* K K - * --- - * K 0 - */ - sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); - /* 0 <= sr <= n_udword_bits - 2 or sr large */ - if (sr > n_udword_bits - 2) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_udword_bits - 1 */ - /* q.all = n.all << (n_utword_bits - sr); */ + // K K + // --- + // K 0 + sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); + // 0 <= sr <= n_udword_bits - 2 or sr large + if (sr > n_udword_bits - 2) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_udword_bits - 1 + // q.all = n.all << (n_utword_bits - sr); + q.s.low = 0; + q.s.high = n.s.low << (n_udword_bits - sr); + // r.all = n.all >> sr; + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + } else /* d.s.low != 0 */ { + if (d.s.high == 0) { + // K X + // --- + // 0 K + if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ { + if (rem) + *rem = n.s.low & (d.s.low - 1); + if (d.s.low == 1) + return n.all; + sr = __builtin_ctzll(d.s.low); + q.s.high = n.s.high >> sr; + q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + return q.all; + } + // K X + // --- + // 0 K + sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) - + __builtin_clzll(n.s.high); + // 2 <= sr <= n_utword_bits - 1 + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + if (sr == n_udword_bits) { + q.s.low = 0; + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else if (sr < n_udword_bits) /* 2 <= sr <= n_udword_bits - 1 */ { q.s.low = 0; q.s.high = n.s.low << (n_udword_bits - sr); - /* r.all = n.all >> sr; */ r.s.high = n.s.high >> sr; r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + } else /* n_udword_bits + 1 <= sr <= n_utword_bits - 1 */ { + q.s.low = n.s.low << (n_utword_bits - sr); + q.s.high = (n.s.high << (n_utword_bits - sr)) | + (n.s.low >> (sr - n_udword_bits)); + r.s.high = 0; + r.s.low = n.s.high >> (sr - n_udword_bits); + } + } else { + // K X + // --- + // K K + sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); + // 0 <= sr <= n_udword_bits - 1 or sr large + if (sr > n_udword_bits - 1) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_udword_bits + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + q.s.low = 0; + if (sr == n_udword_bits) { + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else { + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + q.s.high = n.s.low << (n_udword_bits - sr); + } } - else /* d.s.low != 0 */ - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 K - */ - if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - *rem = n.s.low & (d.s.low - 1); - if (d.s.low == 1) - return n.all; - sr = __builtin_ctzll(d.s.low); - q.s.high = n.s.high >> sr; - q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - return q.all; - } - /* K X - * --- - * 0 K - */ - sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) - - __builtin_clzll(n.s.high); - /* 2 <= sr <= n_utword_bits - 1 - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - */ - if (sr == n_udword_bits) - { - q.s.low = 0; - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else if (sr < n_udword_bits) // 2 <= sr <= n_udword_bits - 1 - { - q.s.low = 0; - q.s.high = n.s.low << (n_udword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - } - else // n_udword_bits + 1 <= sr <= n_utword_bits - 1 - { - q.s.low = n.s.low << (n_utword_bits - sr); - q.s.high = (n.s.high << (n_utword_bits - sr)) | - (n.s.low >> (sr - n_udword_bits)); - r.s.high = 0; - r.s.low = n.s.high >> (sr - n_udword_bits); - } - } - else - { - /* K X - * --- - * K K - */ - sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); - /*0 <= sr <= n_udword_bits - 1 or sr large */ - if (sr > n_udword_bits - 1) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_udword_bits - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - */ - q.s.low = 0; - if (sr == n_udword_bits) - { - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else - { - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - q.s.high = n.s.low << (n_udword_bits - sr); - } - } - } - /* Not a special case - * q and r are initialized with: - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - * 1 <= sr <= n_utword_bits - 1 - */ - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1)); - r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1)); - q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1)); - q.s.low = (q.s.low << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1); - carry = s & 1; - r.all -= d.all & s; - } - q.all = (q.all << 1) | carry; - if (rem) - *rem = r.all; - return q.all; + } + // Not a special case + // q and r are initialized with: + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + // 1 <= sr <= n_utword_bits - 1 + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1)); + r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1)); + q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1)); + q.s.low = (q.s.low << 1) | carry; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1); + carry = s & 1; + r.all -= d.all & s; + } + q.all = (q.all << 1) | carry; + if (rem) + *rem = r.all; + return q.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/udivsi3.c b/lib/builtins/udivsi3.c index bb720f8c3..18cc96c1b 100644 --- a/lib/builtins/udivsi3.c +++ b/lib/builtins/udivsi3.c @@ -1,68 +1,62 @@ -/* ===-- udivsi3.c - Implement __udivsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivsi3.c - Implement __udivsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -/* This function should not call __divsi3! */ -COMPILER_RT_ABI su_int -__udivsi3(su_int n, su_int d) -{ - const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; - su_int q; - su_int r; - unsigned sr; - /* special cases */ - if (d == 0) - return 0; /* ?! */ - if (n == 0) - return 0; - sr = __builtin_clz(d) - __builtin_clz(n); - /* 0 <= sr <= n_uword_bits - 1 or sr large */ - if (sr > n_uword_bits - 1) /* d > r */ - return 0; - if (sr == n_uword_bits - 1) /* d == 1 */ - return n; - ++sr; - /* 1 <= sr <= n_uword_bits - 1 */ - /* Not a special case */ - q = n << (n_uword_bits - sr); - r = n >> sr; - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r = (r << 1) | (q >> (n_uword_bits - 1)); - q = (q << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const si_int s = (si_int)(d - r - 1) >> (n_uword_bits - 1); - carry = s & 1; - r -= d & s; - } +// This function should not call __divsi3! +COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d) { + const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; + su_int q; + su_int r; + unsigned sr; + // special cases + if (d == 0) + return 0; // ?! + if (n == 0) + return 0; + sr = __builtin_clz(d) - __builtin_clz(n); + // 0 <= sr <= n_uword_bits - 1 or sr large + if (sr > n_uword_bits - 1) // d > r + return 0; + if (sr == n_uword_bits - 1) // d == 1 + return n; + ++sr; + // 1 <= sr <= n_uword_bits - 1 + // Not a special case + q = n << (n_uword_bits - sr); + r = n >> sr; + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r = (r << 1) | (q >> (n_uword_bits - 1)); q = (q << 1) | carry; - return q; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const si_int s = (si_int)(d - r - 1) >> (n_uword_bits - 1); + carry = s & 1; + r -= d & s; + } + q = (q << 1) | carry; + return q; } #if defined(__ARM_EABI__) -AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) COMPILER_RT_ALIAS(__udivsi3); +COMPILER_RT_ALIAS(__udivsi3, __aeabi_uidiv) #endif diff --git a/lib/builtins/udivti3.c b/lib/builtins/udivti3.c index ec94673e2..4c82040b8 100644 --- a/lib/builtins/udivti3.c +++ b/lib/builtins/udivti3.c @@ -1,27 +1,23 @@ -/* ===-- udivti3.c - Implement __udivti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivti3.c - Implement __udivti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI tu_int -__udivti3(tu_int a, tu_int b) -{ - return __udivmodti4(a, b, 0); +COMPILER_RT_ABI tu_int __udivti3(tu_int a, tu_int b) { + return __udivmodti4(a, b, 0); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/umoddi3.c b/lib/builtins/umoddi3.c index d513f080a..965cf8fc0 100644 --- a/lib/builtins/umoddi3.c +++ b/lib/builtins/umoddi3.c @@ -1,25 +1,21 @@ -/* ===-- umoddi3.c - Implement __umoddi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umoddi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umoddi3.c - Implement __umoddi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umoddi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI du_int -__umoddi3(du_int a, du_int b) -{ - du_int r; - __udivmoddi4(a, b, &r); - return r; +COMPILER_RT_ABI du_int __umoddi3(du_int a, du_int b) { + du_int r; + __udivmoddi4(a, b, &r); + return r; } diff --git a/lib/builtins/umodsi3.c b/lib/builtins/umodsi3.c index d5fda4a6a..ce9abcd94 100644 --- a/lib/builtins/umodsi3.c +++ b/lib/builtins/umodsi3.c @@ -1,23 +1,19 @@ -/* ===-- umodsi3.c - Implement __umodsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umodsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umodsi3.c - Implement __umodsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umodsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI su_int -__umodsi3(su_int a, su_int b) -{ - return a - __udivsi3(a, b) * b; +COMPILER_RT_ABI su_int __umodsi3(su_int a, su_int b) { + return a - __udivsi3(a, b) * b; } diff --git a/lib/builtins/umodti3.c b/lib/builtins/umodti3.c index 6d1ca7a8c..8cc5cb6b8 100644 --- a/lib/builtins/umodti3.c +++ b/lib/builtins/umodti3.c @@ -1,29 +1,25 @@ -/* ===-- umodti3.c - Implement __umodti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umodti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umodti3.c - Implement __umodti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umodti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI tu_int -__umodti3(tu_int a, tu_int b) -{ - tu_int r; - __udivmodti4(a, b, &r); - return r; +COMPILER_RT_ABI tu_int __umodti3(tu_int a, tu_int b) { + tu_int r; + __udivmodti4(a, b, &r); + return r; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/unwind-ehabi-helpers.h b/lib/builtins/unwind-ehabi-helpers.h index ccb076597..1b48cdb75 100644 --- a/lib/builtins/unwind-ehabi-helpers.h +++ b/lib/builtins/unwind-ehabi-helpers.h @@ -1,43 +1,40 @@ -/* ===-- arm-ehabi-helpers.h - Supplementary ARM EHABI declarations --------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===--------------------------------------------------------------------=== */ +//===-- arm-ehabi-helpers.h - Supplementary ARM EHABI declarations --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===--------------------------------------------------------------------===// #ifndef UNWIND_EHABI_HELPERS_H #define UNWIND_EHABI_HELPERS_H #include <stdint.h> -/* NOTE: see reasoning for this inclusion below */ +// NOTE: see reasoning for this inclusion below #include <unwind.h> #if !defined(__ARM_EABI_UNWINDER__) -/* - * NOTE: _URC_OK, _URC_FAILURE must be present as preprocessor tokens. This - * allows for a substitution of a constant which can be cast into the - * appropriate enumerated type. This header is expected to always be included - * AFTER unwind.h (which is why it is forcefully included above). This ensures - * that we do not overwrite the token for the enumeration. Subsequent uses of - * the token would be clean to rewrite with constant values. - * - * The typedef redeclaration should be safe. Due to the protection granted to - * us by the `__ARM_EABI_UNWINDER__` above, we are guaranteed that we are in a - * header not vended by gcc. The HP unwinder (being an itanium unwinder) does - * not support EHABI, and the GNU unwinder, derived from the HP unwinder, also - * does not support EHABI as of the introduction of this header. As such, we - * are fairly certain that we are in the LLVM case. Here, _Unwind_State is a - * typedef, and so we can get away with a redeclaration. - * - * Guarded redefinitions of the needed unwind state prevent the redefinition of - * those states. - */ - -#define _URC_OK 0 -#define _URC_FAILURE 9 +// NOTE: _URC_OK, _URC_FAILURE must be present as preprocessor tokens. This +// allows for a substitution of a constant which can be cast into the +// appropriate enumerated type. This header is expected to always be included +// AFTER unwind.h (which is why it is forcefully included above). This ensures +// that we do not overwrite the token for the enumeration. Subsequent uses of +// the token would be clean to rewrite with constant values. +// +// The typedef redeclaration should be safe. Due to the protection granted to +// us by the `__ARM_EABI_UNWINDER__` above, we are guaranteed that we are in a +// header not vended by gcc. The HP unwinder (being an itanium unwinder) does +// not support EHABI, and the GNU unwinder, derived from the HP unwinder, also +// does not support EHABI as of the introduction of this header. As such, we +// are fairly certain that we are in the LLVM case. Here, _Unwind_State is a +// typedef, and so we can get away with a redeclaration. +// +// Guarded redefinitions of the needed unwind state prevent the redefinition of +// those states. + +#define _URC_OK 0 +#define _URC_FAILURE 9 typedef uint32_t _Unwind_State; @@ -52,4 +49,3 @@ typedef uint32_t _Unwind_State; #endif #endif - diff --git a/lib/builtins/x86_64/chkstk.S b/lib/builtins/x86_64/chkstk.S index 4149ac63d..ad7953a11 100644 --- a/lib/builtins/x86_64/chkstk.S +++ b/lib/builtins/x86_64/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/x86_64/chkstk2.S b/lib/builtins/x86_64/chkstk2.S index ac1eb920e..33d10d5b6 100644 --- a/lib/builtins/x86_64/chkstk2.S +++ b/lib/builtins/x86_64/chkstk2.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/x86_64/floatdidf.c b/lib/builtins/x86_64/floatdidf.c index dead0ed42..f83f53a38 100644 --- a/lib/builtins/x86_64/floatdidf.c +++ b/lib/builtins/x86_64/floatdidf.c @@ -1,16 +1,13 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* double __floatdidf(di_int a); */ +// double __floatdidf(di_int a); #if defined(__x86_64__) || defined(_M_X64) #include "../int_lib.h" -double __floatdidf(int64_t a) -{ - return (double)a; -} +double __floatdidf(int64_t a) { return (double)a; } -#endif /* __x86_64__ */ +#endif // __x86_64__ diff --git a/lib/builtins/x86_64/floatdisf.c b/lib/builtins/x86_64/floatdisf.c index 99d5621c6..06c118cfa 100644 --- a/lib/builtins/x86_64/floatdisf.c +++ b/lib/builtins/x86_64/floatdisf.c @@ -1,14 +1,11 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #if defined(__x86_64__) || defined(_M_X64) #include "../int_lib.h" -float __floatdisf(int64_t a) -{ - return (float)a; -} +float __floatdisf(int64_t a) { return (float)a; } -#endif /* __x86_64__ */ +#endif // __x86_64__ diff --git a/lib/builtins/x86_64/floatdixf.c b/lib/builtins/x86_64/floatdixf.c index c01193a82..cf8450ce6 100644 --- a/lib/builtins/x86_64/floatdixf.c +++ b/lib/builtins/x86_64/floatdixf.c @@ -1,16 +1,13 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatdixf(di_int a); */ +// long double __floatdixf(di_int a); #ifdef __x86_64__ #include "../int_lib.h" -long double __floatdixf(int64_t a) -{ - return (long double)a; -} +long double __floatdixf(int64_t a) { return (long double)a; } -#endif /* __i386__ */ +#endif // __i386__ diff --git a/lib/builtins/x86_64/floatundidf.S b/lib/builtins/x86_64/floatundidf.S index 094a68dc3..7f6ef3bbb 100644 --- a/lib/builtins/x86_64/floatundidf.S +++ b/lib/builtins/x86_64/floatundidf.S @@ -1,9 +1,8 @@ //===-- floatundidf.S - Implement __floatundidf for x86_64 ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/x86_64/floatundisf.S b/lib/builtins/x86_64/floatundisf.S index 7c9f75e18..246bdff5a 100644 --- a/lib/builtins/x86_64/floatundisf.S +++ b/lib/builtins/x86_64/floatundisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -23,7 +24,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundisf) js 1f cvtsi2ssq %rdi, %xmm0 ret - + 1: andq %rdi, %rsi shrq %rdi orq %rsi, %rdi diff --git a/lib/builtins/x86_64/floatundixf.S b/lib/builtins/x86_64/floatundixf.S index 28a096b71..9e3bcedcb 100644 --- a/lib/builtins/x86_64/floatundixf.S +++ b/lib/builtins/x86_64/floatundixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -57,7 +58,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundixf) orq REL_ADDR(twop52), %rsi // 2^52 + lo (as a double) movq %rdi, -8(%rsp) movq %rsi, -16(%rsp) - fldl REL_ADDR(twop84_plus_twop52_neg) + fldl REL_ADDR(twop84_plus_twop52_neg) faddl -8(%rsp) // hi - 2^52 (as double extended, no rounding occurs) faddl -16(%rsp) // hi + lo (as double extended) ret diff --git a/lib/cfi/CMakeLists.txt b/lib/cfi/CMakeLists.txt index 463a1fd59..9a641d33a 100644 --- a/lib/cfi/CMakeLists.txt +++ b/lib/cfi/CMakeLists.txt @@ -1,7 +1,9 @@ add_compiler_rt_component(cfi) if(OS_NAME MATCHES "Linux" OR OS_NAME MATCHES "FreeBSD" OR OS_NAME MATCHES "NetBSD") - set(CFI_SOURCES cfi.cc) + set(CFI_SOURCES + cfi.cpp + ) include_directories(..) diff --git a/lib/cfi/cfi.cc b/lib/cfi/cfi.cpp index b0a943759..9c34e2b26 100644 --- a/lib/cfi/cfi.cc +++ b/lib/cfi/cfi.cpp @@ -1,9 +1,8 @@ -//===-------- cfi.cc ------------------------------------------------------===// +//===-------- cfi.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -187,7 +186,7 @@ void ShadowBuilder::Install() { GetShadowSize(), MAP_FIXED); CHECK(res != MAP_FAILED); #else - void *res = MmapFixedOrDie(shadow_, GetShadowSize()); + void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow"); CHECK(res != MAP_FAILED); ::memcpy(&shadow_, &main_shadow, GetShadowSize()); #endif diff --git a/lib/crt/CMakeLists.txt b/lib/crt/CMakeLists.txt new file mode 100644 index 000000000..a82ae75c5 --- /dev/null +++ b/lib/crt/CMakeLists.txt @@ -0,0 +1,91 @@ +add_compiler_rt_component(crt) + +function(check_cxx_section_exists section output) + cmake_parse_arguments(ARG "" "" "SOURCE;FLAGS" ${ARGN}) + if(NOT ARG_SOURCE) + set(ARG_SOURCE "int main() { return 0; }\n") + endif() + + string(RANDOM TARGET_NAME) + set(TARGET_NAME "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/cmTC_${TARGET_NAME}.dir") + file(MAKE_DIRECTORY ${TARGET_NAME}) + + file(WRITE "${TARGET_NAME}/CheckSectionExists.c" "${ARG_SOURCE}\n") + + string(REGEX MATCHALL "<[A-Za-z0-9_]*>" substitutions + ${CMAKE_C_COMPILE_OBJECT}) + + set(try_compile_flags "${ARG_FLAGS}") + if(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_TARGET) + list(APPEND try_compile_flags "-target ${CMAKE_C_COMPILER_TARGET}") + endif() + + string(REPLACE ";" " " extra_flags "${try_compile_flags}") + + set(test_compile_command "${CMAKE_C_COMPILE_OBJECT}") + foreach(substitution ${substitutions}) + if(substitution STREQUAL "<CMAKE_C_COMPILER>") + string(REPLACE "<CMAKE_C_COMPILER>" + "${CMAKE_C_COMPILER}" test_compile_command ${test_compile_command}) + elseif(substitution STREQUAL "<OBJECT>") + string(REPLACE "<OBJECT>" "${TARGET_NAME}/CheckSectionExists.o" + test_compile_command ${test_compile_command}) + elseif(substitution STREQUAL "<SOURCE>") + string(REPLACE "<SOURCE>" "${TARGET_NAME}/CheckSectionExists.c" + test_compile_command ${test_compile_command}) + elseif(substitution STREQUAL "<FLAGS>") + string(REPLACE "<FLAGS>" "${CMAKE_C_FLAGS} ${extra_flags}" + test_compile_command ${test_compile_command}) + else() + string(REPLACE "${substitution}" "" test_compile_command + ${test_compile_command}) + endif() + endforeach() + + string(REPLACE " " ";" test_compile_command "${test_compile_command}") + + execute_process( + COMMAND ${test_compile_command} + RESULT_VARIABLE TEST_RESULT + OUTPUT_VARIABLE TEST_OUTPUT + ERROR_VARIABLE TEST_ERROR + ) + + execute_process( + COMMAND ${CMAKE_OBJDUMP} -h "${TARGET_NAME}/CheckSectionExists.o" + RESULT_VARIABLE CHECK_RESULT + OUTPUT_VARIABLE CHECK_OUTPUT + ERROR_VARIABLE CHECK_ERROR + ) + string(FIND "${CHECK_OUTPUT}" "${section}" SECTION_FOUND) + + if(NOT SECTION_FOUND EQUAL -1) + set(${output} TRUE PARENT_SCOPE) + else() + set(${output} FALSE PARENT_SCOPE) + endif() + + file(REMOVE_RECURSE ${TARGET_NAME}) +endfunction() + +check_cxx_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY + SOURCE "__attribute__((constructor)) void f() {}\nint main() { return 0; }\n") + +append_list_if(COMPILER_RT_HAS_INITFINI_ARRAY -DCRT_HAS_INITFINI_ARRAY CRT_CFLAGS) +append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC CRT_CFLAGS) +append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic CRT_CFLAGS) + +foreach(arch ${CRT_SUPPORTED_ARCH}) + add_compiler_rt_runtime(clang_rt.crtbegin + OBJECT + ARCHS ${arch} + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtbegin.c + CFLAGS ${CRT_CFLAGS} + PARENT_TARGET crt) + add_compiler_rt_runtime(clang_rt.crtend + OBJECT + ARCHS ${arch} + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtend.c + CFLAGS ${CRT_CFLAGS} + PARENT_TARGET crt) +endforeach() diff --git a/lib/crt/crtbegin.c b/lib/crt/crtbegin.c new file mode 100644 index 000000000..cfbe5e520 --- /dev/null +++ b/lib/crt/crtbegin.c @@ -0,0 +1,97 @@ +//===-- crtbegin.c - Start of constructors and destructors ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <stddef.h> + +__attribute__((visibility("hidden"))) void *__dso_handle = &__dso_handle; + +__extension__ static void *__EH_FRAME_LIST__[] + __attribute__((section(".eh_frame"), aligned(sizeof(void *)))) = {}; + +extern void __register_frame_info(const void *, void *) __attribute__((weak)); +extern void *__deregister_frame_info(const void *) __attribute__((weak)); + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); + +static fp __CTOR_LIST__[] + __attribute__((section(".ctors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __CTOR_LIST_END__[]; +#endif + +extern void __cxa_finalize(void *) __attribute__((weak)); + +static void __attribute__((used)) __do_init() { + static _Bool __initialized; + if (__builtin_expect(__initialized, 0)) + return; + __initialized = 1; + + static struct { void *p[8]; } __object; + if (__register_frame_info) + __register_frame_info(__EH_FRAME_LIST__, &__object); + +#ifndef CRT_HAS_INITFINI_ARRAY + const size_t n = __CTOR_LIST_END__ - __CTOR_LIST__ - 1; + for (size_t i = n; i >= 1; i--) __CTOR_LIST__[i](); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".init_array"), + used)) static void (*__init)(void) = __do_init; +#else // CRT_HAS_INITFINI_ARRAY +#if defined(__i386__) || defined(__x86_64__) +asm(".pushsection .init,\"ax\",@progbits\n\t" + "call " __USER_LABEL_PREFIX__ "__do_init\n\t" + ".popsection"); +#elif defined(__arm__) +asm(".pushsection .init,\"ax\",%progbits\n\t" + "bl " __USER_LABEL_PREFIX__ "__do_init\n\t" + ".popsection"); +#endif // CRT_HAS_INITFINI_ARRAY +#endif + +#ifndef CRT_HAS_INITFINI_ARRAY +static fp __DTOR_LIST__[] + __attribute__((section(".dtors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __DTOR_LIST_END__[]; +#endif + +static void __attribute__((used)) __do_fini() { + static _Bool __finalized; + if (__builtin_expect(__finalized, 0)) + return; + __finalized = 1; + + if (__cxa_finalize) + __cxa_finalize(__dso_handle); + +#ifndef CRT_HAS_INITFINI_ARRAY + if (__deregister_frame_info) + __deregister_frame_info(__EH_FRAME_LIST__); + + const size_t n = __DTOR_LIST_END__ - __DTOR_LIST__ - 1; + for (size_t i = 1; i <= n; i++) __DTOR_LIST__[i](); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".fini_array"), + used)) static void (*__fini)(void) = __do_fini; +#else // CRT_HAS_INITFINI_ARRAY +#if defined(__i386__) || defined(__x86_64__) +asm(".pushsection .fini,\"ax\",@progbits\n\t" + "call " __USER_LABEL_PREFIX__ "__do_fini\n\t" + ".popsection"); +#elif defined(__arm__) +asm(".pushsection .fini,\"ax\",%progbits\n\t" + "bl " __USER_LABEL_PREFIX__ "__do_fini\n\t" + ".popsection"); +#endif +#endif // CRT_HAS_INIT_FINI_ARRAY diff --git a/lib/crt/crtend.c b/lib/crt/crtend.c new file mode 100644 index 000000000..ebcc60b89 --- /dev/null +++ b/lib/crt/crtend.c @@ -0,0 +1,22 @@ +//===-- crtend.c - End of constructors and destructors --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <stdint.h> + +// Put 4-byte zero which is the length field in FDE at the end as a terminator. +const int32_t __EH_FRAME_LIST_END__[] + __attribute__((section(".eh_frame"), aligned(sizeof(int32_t)), + visibility("hidden"), used)) = {0}; + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); +fp __CTOR_LIST_END__[] + __attribute__((section(".ctors"), visibility("hidden"), used)) = {0}; +fp __DTOR_LIST_END__[] + __attribute__((section(".dtors"), visibility("hidden"), used)) = {0}; +#endif diff --git a/lib/dfsan/dfsan.cc b/lib/dfsan/dfsan.cc index 585bdceac..7f585a8b1 100644 --- a/lib/dfsan/dfsan.cc +++ b/lib/dfsan/dfsan.cc @@ -1,9 +1,8 @@ //===-- dfsan.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan.h b/lib/dfsan/dfsan.h index 33145deef..d66239121 100644 --- a/lib/dfsan/dfsan.h +++ b/lib/dfsan/dfsan.h @@ -1,9 +1,8 @@ //===-- dfsan.h -------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_custom.cc b/lib/dfsan/dfsan_custom.cc index 022aa9a9a..dc7b81da4 100644 --- a/lib/dfsan/dfsan_custom.cc +++ b/lib/dfsan/dfsan_custom.cc @@ -1,9 +1,8 @@ //===-- dfsan.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_flags.inc b/lib/dfsan/dfsan_flags.inc index 24fbfcb9e..cdd0035c9 100644 --- a/lib/dfsan/dfsan_flags.inc +++ b/lib/dfsan/dfsan_flags.inc @@ -1,9 +1,8 @@ //===-- dfsan_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_interceptors.cc b/lib/dfsan/dfsan_interceptors.cc index 5ecbb43e7..f4b4babc6 100644 --- a/lib/dfsan/dfsan_interceptors.cc +++ b/lib/dfsan/dfsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- dfsan_interceptors.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_platform.h b/lib/dfsan/dfsan_platform.h index 98284bafd..4ff68b9d4 100644 --- a/lib/dfsan/dfsan_platform.h +++ b/lib/dfsan/dfsan_platform.h @@ -1,9 +1,8 @@ //===-- dfsan_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/scripts/build-libc-list.py b/lib/dfsan/scripts/build-libc-list.py index eddb6c07e..40805c019 100755 --- a/lib/dfsan/scripts/build-libc-list.py +++ b/lib/dfsan/scripts/build-libc-list.py @@ -1,10 +1,9 @@ #!/usr/bin/env python #===- lib/dfsan/scripts/build-libc-list.py ---------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # The purpose of this script is to identify every function symbol in a set of diff --git a/lib/esan/CMakeLists.txt b/lib/esan/CMakeLists.txt deleted file mode 100644 index c880971e3..000000000 --- a/lib/esan/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# Build for the EfficiencySanitizer runtime support library. - -add_compiler_rt_component(esan) - -set(ESAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS}) -append_rtti_flag(OFF ESAN_RTL_CFLAGS) - -include_directories(..) - -set(ESAN_SOURCES - esan.cpp - esan_flags.cpp - esan_interface.cpp - esan_interceptors.cpp - esan_linux.cpp - esan_sideline_linux.cpp - esan_sideline_bsd.cpp - cache_frag.cpp - working_set.cpp - working_set_posix.cpp) - -set(ESAN_HEADERS - cache_frag.h - esan.h - esan_circular_buffer.h - esan_flags.h - esan_flags.inc - esan_hashtable.h - esan_interface_internal.h - esan_shadow.h - esan_sideline.h - working_set.h) - -foreach (arch ${ESAN_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.esan - STATIC - ARCHS ${arch} - SOURCES ${ESAN_SOURCES} - $<TARGET_OBJECTS:RTInterception.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}> - ADDITIONAL_HEADERS ${ESAN_HEADERS} - CFLAGS ${ESAN_RTL_CFLAGS}) - add_sanitizer_rt_symbols(clang_rt.esan - ARCHS ${arch} - EXTRA esan.syms.extra) - add_dependencies(esan - clang_rt.esan-${arch} - clang_rt.esan-${arch}-symbols) -endforeach() - -if (COMPILER_RT_INCLUDE_TESTS) - # TODO(bruening): add tests via add_subdirectory(tests) -endif() diff --git a/lib/esan/cache_frag.cpp b/lib/esan/cache_frag.cpp deleted file mode 100644 index 5fa5c7d54..000000000 --- a/lib/esan/cache_frag.cpp +++ /dev/null @@ -1,208 +0,0 @@ -//===-- cache_frag.cpp ----------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// This file contains cache fragmentation-specific code. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_flags.h" -#include "sanitizer_common/sanitizer_addrhashmap.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_placement_new.h" -#include <string.h> - -namespace __esan { - -//===-- Struct field access counter runtime -------------------------------===// - -// This should be kept consistent with LLVM's EfficiencySanitizer StructInfo. -struct StructInfo { - const char *StructName; - u32 Size; - u32 NumFields; - u32 *FieldOffset; // auxiliary struct field info. - u32 *FieldSize; // auxiliary struct field info. - const char **FieldTypeName; // auxiliary struct field info. - u64 *FieldCounters; - u64 *ArrayCounter; - bool hasAuxFieldInfo() { return FieldOffset != nullptr; } -}; - -// This should be kept consistent with LLVM's EfficiencySanitizer CacheFragInfo. -// The tool-specific information per compilation unit (module). -struct CacheFragInfo { - const char *UnitName; - u32 NumStructs; - StructInfo *Structs; -}; - -struct StructCounter { - StructInfo *Struct; - u64 Count; // The total access count of the struct. - u64 Ratio; // Difference ratio for the struct layout access. -}; - -// We use StructHashMap to keep track of an unique copy of StructCounter. -typedef AddrHashMap<StructCounter, 31051> StructHashMap; -struct Context { - StructHashMap StructMap; - u32 NumStructs; - u64 TotalCount; // The total access count of all structs. -}; -static Context *Ctx; - -static void reportStructSummary() { - // FIXME: provide a better struct field access summary report. - Report("%s: total struct field access count = %llu\n", SanitizerToolName, - Ctx->TotalCount); -} - -// FIXME: we are still exploring proper ways to evaluate the difference between -// struct field counts. Currently, we use a simple formula to calculate the -// difference ratio: V1/V2. -static inline u64 computeDifferenceRatio(u64 Val1, u64 Val2) { - if (Val2 > Val1) { - Swap(Val1, Val2); - } - if (Val2 == 0) - Val2 = 1; - return (Val1 / Val2); -} - -static void reportStructCounter(StructHashMap::Handle &Handle) { - const u32 TypePrintLimit = 512; - const char *type, *start, *end; - StructInfo *Struct = Handle->Struct; - // Union field address calculation is done via bitcast instead of GEP, - // so the count for union is always 0. - // We skip the union report to avoid confusion. - if (strncmp(Struct->StructName, "union.", 6) == 0) - return; - // Remove the '.' after class/struct during print. - if (strncmp(Struct->StructName, "class.", 6) == 0) { - type = "class"; - start = &Struct->StructName[6]; - } else { - type = "struct"; - start = &Struct->StructName[7]; - } - // Remove the suffixes with '$' during print. - end = strchr(start, '$'); - CHECK(end != nullptr); - Report(" %s %.*s\n", type, end - start, start); - Report(" size = %u, count = %llu, ratio = %llu, array access = %llu\n", - Struct->Size, Handle->Count, Handle->Ratio, *Struct->ArrayCounter); - if (Struct->hasAuxFieldInfo()) { - for (u32 i = 0; i < Struct->NumFields; ++i) { - Report(" #%2u: offset = %u,\t size = %u," - "\t count = %llu,\t type = %.*s\n", - i, Struct->FieldOffset[i], Struct->FieldSize[i], - Struct->FieldCounters[i], TypePrintLimit, Struct->FieldTypeName[i]); - } - } else { - for (u32 i = 0; i < Struct->NumFields; ++i) { - Report(" #%2u: count = %llu\n", i, Struct->FieldCounters[i]); - } - } -} - -static void computeStructRatio(StructHashMap::Handle &Handle) { - Handle->Ratio = 0; - Handle->Count = Handle->Struct->FieldCounters[0]; - for (u32 i = 1; i < Handle->Struct->NumFields; ++i) { - Handle->Count += Handle->Struct->FieldCounters[i]; - Handle->Ratio += computeDifferenceRatio( - Handle->Struct->FieldCounters[i - 1], Handle->Struct->FieldCounters[i]); - } - Ctx->TotalCount += Handle->Count; - if (Handle->Ratio >= (u64)getFlags()->report_threshold || - (Verbosity() >= 1 && Handle->Count > 0)) - reportStructCounter(Handle); -} - -static void registerStructInfo(CacheFragInfo *CacheFrag) { - for (u32 i = 0; i < CacheFrag->NumStructs; ++i) { - StructInfo *Struct = &CacheFrag->Structs[i]; - StructHashMap::Handle H(&Ctx->StructMap, (uptr)Struct->FieldCounters); - if (H.created()) { - VPrintf(2, " Register %s: %u fields\n", Struct->StructName, - Struct->NumFields); - H->Struct = Struct; - ++Ctx->NumStructs; - } else { - VPrintf(2, " Duplicated %s: %u fields\n", Struct->StructName, - Struct->NumFields); - } - } -} - -static void unregisterStructInfo(CacheFragInfo *CacheFrag) { - // FIXME: if the library is unloaded before finalizeCacheFrag, we should - // collect the result for later report. - for (u32 i = 0; i < CacheFrag->NumStructs; ++i) { - StructInfo *Struct = &CacheFrag->Structs[i]; - StructHashMap::Handle H(&Ctx->StructMap, (uptr)Struct->FieldCounters, true); - if (H.exists()) { - VPrintf(2, " Unregister %s: %u fields\n", Struct->StructName, - Struct->NumFields); - // FIXME: we should move this call to finalizeCacheFrag once we can - // iterate over the hash map there. - computeStructRatio(H); - --Ctx->NumStructs; - } else { - VPrintf(2, " Duplicated %s: %u fields\n", Struct->StructName, - Struct->NumFields); - } - } - static bool Reported = false; - if (Ctx->NumStructs == 0 && !Reported) { - Reported = true; - reportStructSummary(); - } -} - -//===-- Init/exit functions -----------------------------------------------===// - -void processCacheFragCompilationUnitInit(void *Ptr) { - CacheFragInfo *CacheFrag = (CacheFragInfo *)Ptr; - VPrintf(2, "in esan::%s: %s with %u class(es)/struct(s)\n", __FUNCTION__, - CacheFrag->UnitName, CacheFrag->NumStructs); - registerStructInfo(CacheFrag); -} - -void processCacheFragCompilationUnitExit(void *Ptr) { - CacheFragInfo *CacheFrag = (CacheFragInfo *)Ptr; - VPrintf(2, "in esan::%s: %s with %u class(es)/struct(s)\n", __FUNCTION__, - CacheFrag->UnitName, CacheFrag->NumStructs); - unregisterStructInfo(CacheFrag); -} - -void initializeCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - // We use placement new to initialize Ctx before C++ static initializaion. - // We make CtxMem 8-byte aligned for atomic operations in AddrHashMap. - static u64 CtxMem[sizeof(Context) / sizeof(u64) + 1]; - Ctx = new (CtxMem) Context(); - Ctx->NumStructs = 0; -} - -int finalizeCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - return 0; -} - -void reportCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - // FIXME: Not yet implemented. We need to iterate over all of the - // compilation unit data. -} - -} // namespace __esan diff --git a/lib/esan/cache_frag.h b/lib/esan/cache_frag.h deleted file mode 100644 index 646d3f85e..000000000 --- a/lib/esan/cache_frag.h +++ /dev/null @@ -1,29 +0,0 @@ -//===-- cache_frag.h --------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Header for cache-fragmentation-specific code. -//===----------------------------------------------------------------------===// - -#ifndef CACHE_FRAG_H -#define CACHE_FRAG_H - -namespace __esan { - -void processCacheFragCompilationUnitInit(void *Ptr); -void processCacheFragCompilationUnitExit(void *Ptr); - -void initializeCacheFrag(); -int finalizeCacheFrag(); -void reportCacheFrag(); - -} // namespace __esan - -#endif // CACHE_FRAG_H diff --git a/lib/esan/esan.cpp b/lib/esan/esan.cpp deleted file mode 100644 index 44b8032ca..000000000 --- a/lib/esan/esan.cpp +++ /dev/null @@ -1,278 +0,0 @@ -//===-- esan.cpp ----------------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Main file (entry points) for the Esan run-time. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_flags.h" -#include "esan_interface_internal.h" -#include "esan_shadow.h" -#include "cache_frag.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flag_parser.h" -#include "sanitizer_common/sanitizer_flags.h" -#include "working_set.h" - -// See comment below. -extern "C" { -extern void __cxa_atexit(void (*function)(void)); -} - -namespace __esan { - -bool EsanIsInitialized; -bool EsanDuringInit; -ShadowMapping Mapping; - -// Different tools use different scales within the same shadow mapping scheme. -// The scale used here must match that used by the compiler instrumentation. -// This array is indexed by the ToolType enum. -static const uptr ShadowScale[] = { - 0, // ESAN_None. - 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2. - 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6. -}; - -// We are combining multiple performance tuning tools under the umbrella of -// one EfficiencySanitizer super-tool. Most of our tools have very similar -// memory access instrumentation, shadow memory mapping, libc interception, -// etc., and there is typically more shared code than distinct code. -// -// We are not willing to dispatch on tool dynamically in our fastpath -// instrumentation: thus, which tool to use is a static option selected -// at compile time and passed to __esan_init(). -// -// We are willing to pay the overhead of tool dispatch in the slowpath to more -// easily share code. We expect to only come here rarely. -// If this becomes a performance hit, we can add separate interface -// routines for each subtool (e.g., __esan_cache_frag_aligned_load_4). -// But for libc interceptors, we'll have to do one of the following: -// A) Add multiple-include support to sanitizer_common_interceptors.inc, -// instantiate it separately for each tool, and call the selected -// tool's intercept setup code. -// B) Build separate static runtime libraries, one for each tool. -// C) Completely split the tools into separate sanitizers. - -void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite) { - VPrintf(3, "in esan::%s %p: %c %p %d\n", __FUNCTION__, PC, - IsWrite ? 'w' : 'r', Addr, Size); - if (__esan_which_tool == ESAN_CacheFrag) { - // TODO(bruening): add shadow mapping and update shadow bits here. - // We'll move this to cache_frag.cpp once we have something. - } else if (__esan_which_tool == ESAN_WorkingSet) { - processRangeAccessWorkingSet(PC, Addr, Size, IsWrite); - } -} - -bool processSignal(int SigNum, void (*Handler)(int), void (**Result)(int)) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSignal(SigNum, Handler, Result); - return true; -} - -bool processSigaction(int SigNum, const void *Act, void *OldAct) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSigaction(SigNum, Act, OldAct); - return true; -} - -bool processSigprocmask(int How, void *Set, void *OldSet) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSigprocmask(How, Set, OldSet); - return true; -} - -#if SANITIZER_DEBUG -static bool verifyShadowScheme() { - // Sanity checks for our shadow mapping scheme. - uptr AppStart, AppEnd; - if (Verbosity() >= 3) { - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - VPrintf(3, "App #%d: [%zx-%zx) (%zuGB)\n", i, AppStart, AppEnd, - (AppEnd - AppStart) >> 30); - } - } - for (int Scale = 0; Scale < 8; ++Scale) { - Mapping.initialize(Scale); - if (Verbosity() >= 3) { - VPrintf(3, "\nChecking scale %d\n", Scale); - uptr ShadowStart, ShadowEnd; - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(3, "Shadow #%d: [%zx-%zx) (%zuGB)\n", i, ShadowStart, - ShadowEnd, (ShadowEnd - ShadowStart) >> 30); - } - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(3, "Shadow(Shadow) #%d: [%zx-%zx)\n", i, - appToShadow(ShadowStart), appToShadow(ShadowEnd - 1)+1); - } - } - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - DCHECK(isAppMem(AppStart)); - DCHECK(!isAppMem(AppStart - 1)); - DCHECK(isAppMem(AppEnd - 1)); - DCHECK(!isAppMem(AppEnd)); - DCHECK(!isShadowMem(AppStart)); - DCHECK(!isShadowMem(AppEnd - 1)); - DCHECK(isShadowMem(appToShadow(AppStart))); - DCHECK(isShadowMem(appToShadow(AppEnd - 1))); - // Double-shadow checks. - DCHECK(!isShadowMem(appToShadow(appToShadow(AppStart)))); - DCHECK(!isShadowMem(appToShadow(appToShadow(AppEnd - 1)))); - } - // Ensure no shadow regions overlap each other. - uptr ShadowAStart, ShadowBStart, ShadowAEnd, ShadowBEnd; - for (int i = 0; getShadowRegion(i, &ShadowAStart, &ShadowAEnd); ++i) { - for (int j = 0; getShadowRegion(j, &ShadowBStart, &ShadowBEnd); ++j) { - DCHECK(i == j || ShadowAStart >= ShadowBEnd || - ShadowAEnd <= ShadowBStart); - } - } - } - return true; -} -#endif - -uptr VmaSize; - -static void initializeShadow() { - verifyAddressSpace(); - - // This is based on the assumption that the intial stack is always allocated - // in the topmost segment of the user address space and the assumption - // holds true on all the platforms currently supported. - VmaSize = - (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); - - DCHECK(verifyShadowScheme()); - - Mapping.initialize(ShadowScale[__esan_which_tool]); - - VPrintf(1, "Shadow scale=%d offset=%p\n", Mapping.Scale, Mapping.Offset); - - uptr ShadowStart, ShadowEnd; - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(1, "Shadow #%d: [%zx-%zx) (%zuGB)\n", i, ShadowStart, ShadowEnd, - (ShadowEnd - ShadowStart) >> 30); - - uptr Map = 0; - if (__esan_which_tool == ESAN_WorkingSet) { - // We want to identify all shadow pages that are touched so we start - // out inaccessible. - Map = (uptr)MmapFixedNoAccess(ShadowStart, ShadowEnd- ShadowStart, - "shadow"); - } else { - if (MmapFixedNoReserve(ShadowStart, ShadowEnd - ShadowStart, "shadow")) - Map = ShadowStart; - } - if (Map != ShadowStart) { - Printf("FATAL: EfficiencySanitizer failed to map its shadow memory.\n"); - Die(); - } - - if (common_flags()->no_huge_pages_for_shadow) - NoHugePagesInRegion(ShadowStart, ShadowEnd - ShadowStart); - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(ShadowStart, ShadowEnd - ShadowStart); - - // TODO: Call MmapNoAccess() on in-between regions. - } -} - -void initializeLibrary(ToolType Tool) { - // We assume there is only one thread during init, but we need to - // guard against double-init when we're (re-)called from an - // early interceptor. - if (EsanIsInitialized || EsanDuringInit) - return; - EsanDuringInit = true; - CHECK(Tool == __esan_which_tool); - SanitizerToolName = "EfficiencySanitizer"; - CacheBinaryName(); - initializeFlags(); - - // Intercepting libc _exit or exit via COMMON_INTERCEPTOR_ON_EXIT only - // finalizes on an explicit exit call by the app. To handle a normal - // exit we register an atexit handler. - ::__cxa_atexit((void (*)())finalizeLibrary); - - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool <= ESAN_None || __esan_which_tool >= ESAN_Max) { - Printf("ERROR: unknown tool %d requested\n", __esan_which_tool); - Die(); - } - - initializeShadow(); - if (__esan_which_tool == ESAN_WorkingSet) - initializeShadowWorkingSet(); - - initializeInterceptors(); - - if (__esan_which_tool == ESAN_CacheFrag) { - initializeCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - initializeWorkingSet(); - } - - EsanIsInitialized = true; - EsanDuringInit = false; -} - -int finalizeLibrary() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - return finalizeCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - return finalizeWorkingSet(); - } - return 0; -} - -void reportResults() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - return reportCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - return reportWorkingSet(); - } -} - -void processCompilationUnitInit(void *Ptr) { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - DCHECK(Ptr != nullptr); - processCacheFragCompilationUnitInit(Ptr); - } else { - DCHECK(Ptr == nullptr); - } -} - -// This is called when the containing module is unloaded. -// For the main executable module, this is called after finalizeLibrary. -void processCompilationUnitExit(void *Ptr) { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - DCHECK(Ptr != nullptr); - processCacheFragCompilationUnitExit(Ptr); - } else { - DCHECK(Ptr == nullptr); - } -} - -unsigned int getSampleCount() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_WorkingSet) { - return getSampleCountWorkingSet(); - } - return 0; -} - -} // namespace __esan diff --git a/lib/esan/esan.h b/lib/esan/esan.h deleted file mode 100644 index e73b21e56..000000000 --- a/lib/esan/esan.h +++ /dev/null @@ -1,61 +0,0 @@ -//===-- esan.h --------------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Main internal esan header file. -// -// Ground rules: -// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static -// function-scope locals) -// - All functions/classes/etc reside in namespace __esan, except for those -// declared in esan_interface_internal.h. -// - Platform-specific files should be used instead of ifdefs (*). -// - No system headers included in header files (*). -// - Platform specific headers included only into platform-specific files (*). -// -// (*) Except when inlining is critical for performance. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_H -#define ESAN_H - -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include "esan_interface_internal.h" - -namespace __esan { - -extern bool EsanIsInitialized; -extern bool EsanDuringInit; -extern uptr VmaSize; - -void initializeLibrary(ToolType Tool); -int finalizeLibrary(); -void reportResults(); -unsigned int getSampleCount(); -// Esan creates the variable per tool per compilation unit at compile time -// and passes its pointer Ptr to the runtime library. -void processCompilationUnitInit(void *Ptr); -void processCompilationUnitExit(void *Ptr); -void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite); -void initializeInterceptors(); - -// Platform-dependent routines. -void verifyAddressSpace(); -bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags); -uptr checkMmapResult(uptr Addr, SIZE_T Size); -// The return value indicates whether to call the real version or not. -bool processSignal(int SigNum, void (*Handler)(int), void (**Result)(int)); -bool processSigaction(int SigNum, const void *Act, void *OldAct); -bool processSigprocmask(int How, void *Set, void *OldSet); - -} // namespace __esan - -#endif // ESAN_H diff --git a/lib/esan/esan.syms.extra b/lib/esan/esan.syms.extra deleted file mode 100644 index d6397d4c3..000000000 --- a/lib/esan/esan.syms.extra +++ /dev/null @@ -1,4 +0,0 @@ -__esan_init -__esan_exit -__esan_aligned* -__esan_unaligned* diff --git a/lib/esan/esan_circular_buffer.h b/lib/esan/esan_circular_buffer.h deleted file mode 100644 index 9ce102d04..000000000 --- a/lib/esan/esan_circular_buffer.h +++ /dev/null @@ -1,96 +0,0 @@ -//===-- esan_circular_buffer.h ----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Circular buffer data structure. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_common.h" - -namespace __esan { - -// A circular buffer for POD data whose memory is allocated using mmap. -// There are two usage models: one is to use initialize/free (for global -// instances) and the other is to use placement new with the -// constructor and to call the destructor or free (they are equivalent). -template<typename T> -class CircularBuffer { - public: - // To support global instances we cannot initialize any field in the - // default constructor. - explicit CircularBuffer() {} - CircularBuffer(uptr BufferCapacity) { - initialize(BufferCapacity); - WasConstructed = true; - } - ~CircularBuffer() { - if (WasConstructed) // Else caller will call free() explicitly. - free(); - } - void initialize(uptr BufferCapacity) { - Capacity = BufferCapacity; - // MmapOrDie rounds up to the page size for us. - Data = (T *)MmapOrDie(Capacity * sizeof(T), "CircularBuffer"); - StartIdx = 0; - Count = 0; - WasConstructed = false; - } - void free() { - UnmapOrDie(Data, Capacity * sizeof(T)); - } - T &operator[](uptr Idx) { - CHECK_LT(Idx, Count); - uptr ArrayIdx = (StartIdx + Idx) % Capacity; - return Data[ArrayIdx]; - } - const T &operator[](uptr Idx) const { - CHECK_LT(Idx, Count); - uptr ArrayIdx = (StartIdx + Idx) % Capacity; - return Data[ArrayIdx]; - } - void push_back(const T &Item) { - CHECK_GT(Capacity, 0); - uptr ArrayIdx = (StartIdx + Count) % Capacity; - Data[ArrayIdx] = Item; - if (Count < Capacity) - ++Count; - else - StartIdx = (StartIdx + 1) % Capacity; - } - T &back() { - CHECK_GT(Count, 0); - uptr ArrayIdx = (StartIdx + Count - 1) % Capacity; - return Data[ArrayIdx]; - } - void pop_back() { - CHECK_GT(Count, 0); - --Count; - } - uptr size() const { - return Count; - } - void clear() { - StartIdx = 0; - Count = 0; - } - bool empty() const { return size() == 0; } - - private: - CircularBuffer(const CircularBuffer&); - void operator=(const CircularBuffer&); - - bool WasConstructed; - T *Data; - uptr Capacity; - uptr StartIdx; - uptr Count; -}; - -} // namespace __esan diff --git a/lib/esan/esan_flags.cpp b/lib/esan/esan_flags.cpp deleted file mode 100644 index c90bf2493..000000000 --- a/lib/esan/esan_flags.cpp +++ /dev/null @@ -1,60 +0,0 @@ -//===-- esan_flags.cc -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan flag parsing logic. -//===----------------------------------------------------------------------===// - -#include "esan_flags.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flag_parser.h" -#include "sanitizer_common/sanitizer_flags.h" - -using namespace __sanitizer; - -namespace __esan { - -static const char EsanOptsEnv[] = "ESAN_OPTIONS"; - -Flags EsanFlagsDontUseDirectly; - -void Flags::setDefaults() { -#define ESAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; -#include "esan_flags.inc" -#undef ESAN_FLAG -} - -static void registerEsanFlags(FlagParser *Parser, Flags *F) { -#define ESAN_FLAG(Type, Name, DefaultValue, Description) \ - RegisterFlag(Parser, #Name, Description, &F->Name); -#include "esan_flags.inc" -#undef ESAN_FLAG -} - -void initializeFlags() { - SetCommonFlagsDefaults(); - Flags *F = getFlags(); - F->setDefaults(); - - FlagParser Parser; - registerEsanFlags(&Parser, F); - RegisterCommonFlags(&Parser); - Parser.ParseString(GetEnv(EsanOptsEnv)); - - InitializeCommonFlags(); - if (Verbosity()) - ReportUnrecognizedFlags(); - if (common_flags()->help) - Parser.PrintFlagDescriptions(); - - __sanitizer_set_report_path(common_flags()->log_path); -} - -} // namespace __esan diff --git a/lib/esan/esan_flags.h b/lib/esan/esan_flags.h deleted file mode 100644 index c8f4ef5ab..000000000 --- a/lib/esan/esan_flags.h +++ /dev/null @@ -1,41 +0,0 @@ -//===-- esan_flags.h --------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan runtime flags. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_FLAGS_H -#define ESAN_FLAGS_H - -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_flag_parser.h" - -namespace __esan { - -class Flags { -public: -#define ESAN_FLAG(Type, Name, DefaultValue, Description) Type Name; -#include "esan_flags.inc" -#undef ESAN_FLAG - - void setDefaults(); -}; - -extern Flags EsanFlagsDontUseDirectly; -inline Flags *getFlags() { - return &EsanFlagsDontUseDirectly; -} - -void initializeFlags(); - -} // namespace __esan - -#endif // ESAN_FLAGS_H diff --git a/lib/esan/esan_flags.inc b/lib/esan/esan_flags.inc deleted file mode 100644 index 5687caca2..000000000 --- a/lib/esan/esan_flags.inc +++ /dev/null @@ -1,56 +0,0 @@ -//===-- esan_flags.inc ------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Esan runtime flags. -// -//===----------------------------------------------------------------------===// - -#ifndef ESAN_FLAG -# error "Define ESAN_FLAG prior to including this file!" -#endif - -// ESAN_FLAG(Type, Name, DefaultValue, Description) -// See COMMON_FLAG in sanitizer_flags.inc for more details. - -//===----------------------------------------------------------------------===// -// Cross-tool options -//===----------------------------------------------------------------------===// - -ESAN_FLAG(int, cache_line_size, 64, - "The number of bytes in a cache line. For the working-set tool, this " - "cannot be changed without also changing the compiler " - "instrumentation.") - -//===----------------------------------------------------------------------===// -// Working set tool options -//===----------------------------------------------------------------------===// - -ESAN_FLAG(bool, record_snapshots, true, - "Working set tool: whether to sample snapshots during a run.") - -// Typical profiling uses a 10ms timer. Our snapshots take some work -// to scan memory so we reduce to 20ms. -// To disable samples, turn off record_snapshots. -ESAN_FLAG(int, sample_freq, 20, - "Working set tool: sampling frequency in milliseconds.") - -// This controls the difference in frequency between each successive series -// of snapshots. There are 8 in total, with number 0 using sample_freq. -// Number N samples number N-1 every (1 << snapshot_step) instance of N-1. -ESAN_FLAG(int, snapshot_step, 2, "Working set tool: the log of the sampling " - "performed for the next-higher-frequency snapshot series.") - -//===----------------------------------------------------------------------===// -// Cache Fragmentation tool options -//===----------------------------------------------------------------------===// - -// The difference information of a struct is reported if the struct's difference -// score is greater than the report_threshold. -ESAN_FLAG(int, report_threshold, 1<<10, "Cache-frag tool: the struct difference" - " score threshold for reporting.") diff --git a/lib/esan/esan_hashtable.h b/lib/esan/esan_hashtable.h deleted file mode 100644 index 7bd829740..000000000 --- a/lib/esan/esan_hashtable.h +++ /dev/null @@ -1,381 +0,0 @@ -//===-- esan_hashtable.h ----------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Generic resizing hashtable. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_allocator_internal.h" -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_mutex.h" -#include <stddef.h> - -namespace __esan { - -//===----------------------------------------------------------------------===// -// Default hash and comparison functions -//===----------------------------------------------------------------------===// - -template <typename T> struct DefaultHash { - size_t operator()(const T &Key) const { - return (size_t)Key; - } -}; - -template <typename T> struct DefaultEqual { - bool operator()(const T &Key1, const T &Key2) const { - return Key1 == Key2; - } -}; - -//===----------------------------------------------------------------------===// -// HashTable declaration -//===----------------------------------------------------------------------===// - -// A simple resizing and mutex-locked hashtable. -// -// If the default hash functor is used, KeyTy must have an operator size_t(). -// If the default comparison functor is used, KeyTy must have an operator ==. -// -// By default all operations are internally-synchronized with a mutex, with no -// synchronization for payloads once hashtable functions return. If -// ExternalLock is set to true, the caller should call the lock() and unlock() -// routines around all hashtable operations and subsequent manipulation of -// payloads. -template <typename KeyTy, typename DataTy, bool ExternalLock = false, - typename HashFuncTy = DefaultHash<KeyTy>, - typename EqualFuncTy = DefaultEqual<KeyTy> > -class HashTable { -public: - // InitialCapacity must be a power of 2. - // ResizeFactor must be between 1 and 99 and indicates the - // maximum percentage full that the table should ever be. - HashTable(u32 InitialCapacity = 2048, u32 ResizeFactor = 70); - ~HashTable(); - bool lookup(const KeyTy &Key, DataTy &Payload); // Const except for Mutex. - bool add(const KeyTy &Key, const DataTy &Payload); - bool remove(const KeyTy &Key); - u32 size(); // Const except for Mutex. - // If the table is internally-synchronized, this lock must not be held - // while a hashtable function is called as it will deadlock: the lock - // is not recursive. This is meant for use with externally-synchronized - // tables or with an iterator. - void lock(); - void unlock(); - -private: - struct HashEntry { - KeyTy Key; - DataTy Payload; - HashEntry *Next; - }; - -public: - struct HashPair { - HashPair(KeyTy Key, DataTy Data) : Key(Key), Data(Data) {} - KeyTy Key; - DataTy Data; - }; - - // This iterator does not perform any synchronization. - // It expects the caller to lock the table across the whole iteration. - // Calling HashTable functions while using the iterator is not supported. - // The iterator returns copies of the keys and data. - class iterator { - public: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table); - iterator(const iterator &Src) = default; - iterator &operator=(const iterator &Src) = default; - HashPair operator*(); - iterator &operator++(); - iterator &operator++(int); - bool operator==(const iterator &Cmp) const; - bool operator!=(const iterator &Cmp) const; - - private: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table, - int Idx); - friend HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>; - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table; - int Idx; - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::HashEntry - *Entry; - }; - - // No erase or insert iterator supported - iterator begin(); - iterator end(); - -private: - void resize(); - - HashEntry **Table; - u32 Capacity; - u32 Entries; - const u32 ResizeFactor; - BlockingMutex Mutex; - const HashFuncTy HashFunc; - const EqualFuncTy EqualFunc; -}; - -//===----------------------------------------------------------------------===// -// Hashtable implementation -//===----------------------------------------------------------------------===// - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::HashTable( - u32 InitialCapacity, u32 ResizeFactor) - : Capacity(InitialCapacity), Entries(0), ResizeFactor(ResizeFactor), - HashFunc(HashFuncTy()), EqualFunc(EqualFuncTy()) { - CHECK(IsPowerOfTwo(Capacity)); - CHECK(ResizeFactor >= 1 && ResizeFactor <= 99); - Table = (HashEntry **)InternalAlloc(Capacity * sizeof(HashEntry *)); - internal_memset(Table, 0, Capacity * sizeof(HashEntry *)); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::~HashTable() { - for (u32 i = 0; i < Capacity; ++i) { - HashEntry *Entry = Table[i]; - while (Entry != nullptr) { - HashEntry *Next = Entry->Next; - Entry->Payload.~DataTy(); - InternalFree(Entry); - Entry = Next; - } - } - InternalFree(Table); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -u32 HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::size() { - u32 Res; - if (!ExternalLock) - Mutex.Lock(); - Res = Entries; - if (!ExternalLock) - Mutex.Unlock(); - return Res; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::lookup( - const KeyTy &Key, DataTy &Payload) { - if (!ExternalLock) - Mutex.Lock(); - bool Found = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - for (; Entry != nullptr; Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Payload = Entry->Payload; - Found = true; - break; - } - } - if (!ExternalLock) - Mutex.Unlock(); - return Found; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::resize() { - if (!ExternalLock) - Mutex.CheckLocked(); - size_t OldCapacity = Capacity; - HashEntry **OldTable = Table; - Capacity *= 2; - Table = (HashEntry **)InternalAlloc(Capacity * sizeof(HashEntry *)); - internal_memset(Table, 0, Capacity * sizeof(HashEntry *)); - // Re-hash - for (u32 i = 0; i < OldCapacity; ++i) { - HashEntry *OldEntry = OldTable[i]; - while (OldEntry != nullptr) { - HashEntry *Next = OldEntry->Next; - size_t Hash = HashFunc(OldEntry->Key) % Capacity; - OldEntry->Next = Table[Hash]; - Table[Hash] = OldEntry; - OldEntry = Next; - } - } -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::add( - const KeyTy &Key, const DataTy &Payload) { - if (!ExternalLock) - Mutex.Lock(); - bool Exists = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - for (; Entry != nullptr; Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Exists = true; - break; - } - } - if (!Exists) { - Entries++; - if (Entries * 100 >= Capacity * ResizeFactor) { - resize(); - Hash = HashFunc(Key) % Capacity; - } - HashEntry *Add = (HashEntry *)InternalAlloc(sizeof(*Add)); - Add->Key = Key; - Add->Payload = Payload; - Add->Next = Table[Hash]; - Table[Hash] = Add; - } - if (!ExternalLock) - Mutex.Unlock(); - return !Exists; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::remove( - const KeyTy &Key) { - if (!ExternalLock) - Mutex.Lock(); - bool Found = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - HashEntry *Prev = nullptr; - for (; Entry != nullptr; Prev = Entry, Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Found = true; - Entries--; - if (Prev == nullptr) - Table[Hash] = Entry->Next; - else - Prev->Next = Entry->Next; - Entry->Payload.~DataTy(); - InternalFree(Entry); - break; - } - } - if (!ExternalLock) - Mutex.Unlock(); - return Found; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::lock() { - Mutex.Lock(); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::unlock() { - Mutex.Unlock(); -} - -//===----------------------------------------------------------------------===// -// Iterator implementation -//===----------------------------------------------------------------------===// - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table) - : Table(Table), Idx(-1), Entry(nullptr) { - operator++(); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table, - int Idx) - : Table(Table), Idx(Idx), Entry(nullptr) { - CHECK(Idx >= (int)Table->Capacity); // Only used to create end(). -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::HashPair - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator*() { - CHECK(Idx >= 0 && Idx < (int)Table->Capacity); - CHECK(Entry != nullptr); - return HashPair(Entry->Key, Entry->Payload); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator & - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator++() { - if (Entry != nullptr) - Entry = Entry->Next; - while (Entry == nullptr) { - ++Idx; - if (Idx >= (int)Table->Capacity) - break; // At end(). - Entry = Table->Table[Idx]; - } - return *this; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator & - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator++(int) { - iterator Temp(*this); - operator++(); - return Temp; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: -operator==(const iterator &Cmp) const { - return Cmp.Table == Table && Cmp.Idx == Idx && Cmp.Entry == Entry; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: -operator!=(const iterator &Cmp) const { - return Cmp.Table != Table || Cmp.Idx != Idx || Cmp.Entry != Entry; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::begin() { - return iterator(this); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::end() { - return iterator(this, Capacity); -} - -} // namespace __esan diff --git a/lib/esan/esan_interceptors.cpp b/lib/esan/esan_interceptors.cpp deleted file mode 100644 index 833faa2cd..000000000 --- a/lib/esan/esan_interceptors.cpp +++ /dev/null @@ -1,512 +0,0 @@ -//===-- esan_interceptors.cpp ---------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Interception routines for the esan run-time. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_shadow.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_libc.h" -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_stacktrace.h" - -using namespace __esan; // NOLINT - -#define CUR_PC() (StackTrace::GetCurrentPc()) - -//===----------------------------------------------------------------------===// -// Interception via sanitizer common interceptors -//===----------------------------------------------------------------------===// - -// Get the per-platform defines for what is possible to intercept -#include "sanitizer_common/sanitizer_platform_interceptors.h" - -DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) - -// TODO(bruening): tsan disables several interceptors (getpwent, etc.) claiming -// that interception is a perf hit: should we do the same? - -// We have no need to intercept: -#undef SANITIZER_INTERCEPT_TLS_GET_ADDR - -// TODO(bruening): the common realpath interceptor assumes malloc is -// intercepted! We should try to parametrize that, though we'll -// intercept malloc soon ourselves and can then remove this undef. -#undef SANITIZER_INTERCEPT_REALPATH - -// We provide our own version: -#undef SANITIZER_INTERCEPT_SIGPROCMASK - -#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!EsanIsInitialized) - -#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) -#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ - INTERCEPT_FUNCTION_VER(name, ver) - -// We must initialize during early interceptors, to support tcmalloc. -// This means that for some apps we fully initialize prior to -// __esan_init() being called. -// We currently do not use ctx. -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - do { \ - if (UNLIKELY(COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)) { \ - if (!UNLIKELY(EsanDuringInit)) \ - initializeLibrary(__esan_which_tool); \ - return REAL(func)(__VA_ARGS__); \ - } \ - ctx = nullptr; \ - (void)ctx; \ - } while (false) - -#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ - COMMON_INTERCEPTOR_ENTER(ctx, func, __VA_ARGS__) - -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - processRangeAccess(CUR_PC(), (uptr)ptr, size, true) - -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - processRangeAccess(CUR_PC(), (uptr)ptr, size, false) - -// This is only called if the app explicitly calls exit(), not on -// a normal exit. -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) finalizeLibrary() - -#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ - do { \ - (void)(ctx); \ - (void)(file); \ - (void)(path); \ - } while (false) -#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ - do { \ - (void)(ctx); \ - (void)(file); \ - } while (false) -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ - do { \ - (void)(filename); \ - (void)(handle); \ - } while (false) -#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ - do { \ - (void)(ctx); \ - (void)(u); \ - } while (false) -#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ - do { \ - (void)(ctx); \ - (void)(u); \ - } while (false) -#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ - do { \ - (void)(ctx); \ - (void)(path); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - (void)(newfd); \ - } while (false) -#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - do { \ - (void)(ctx); \ - (void)(name); \ - } while (false) -#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - do { \ - (void)(ctx); \ - (void)(thread); \ - (void)(name); \ - } while (false) -#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ - do { \ - (void)(ctx); \ - (void)(msg); \ - } while (false) -#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ - do { \ - } while (false) - -#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ - off) \ - do { \ - if (!fixMmapAddr(&addr, sz, flags)) \ - return (void *)-1; \ - void *result = REAL(mmap)(addr, sz, prot, flags, fd, off); \ - return (void *)checkMmapResult((uptr)result, sz); \ - } while (false) - -#include "sanitizer_common/sanitizer_common_interceptors.inc" - -//===----------------------------------------------------------------------===// -// Syscall interception -//===----------------------------------------------------------------------===// - -// We want the caller's PC b/c unlike the other function interceptors these -// are separate pre and post functions called around the app's syscall(). - -#define COMMON_SYSCALL_PRE_READ_RANGE(ptr, size) \ - processRangeAccess(GET_CALLER_PC(), (uptr)ptr, size, false) - -#define COMMON_SYSCALL_PRE_WRITE_RANGE(ptr, size) \ - do { \ - (void)(ptr); \ - (void)(size); \ - } while (false) - -#define COMMON_SYSCALL_POST_READ_RANGE(ptr, size) \ - do { \ - (void)(ptr); \ - (void)(size); \ - } while (false) - -// The actual amount written is in post, not pre. -#define COMMON_SYSCALL_POST_WRITE_RANGE(ptr, size) \ - processRangeAccess(GET_CALLER_PC(), (uptr)ptr, size, true) - -#define COMMON_SYSCALL_ACQUIRE(addr) \ - do { \ - (void)(addr); \ - } while (false) -#define COMMON_SYSCALL_RELEASE(addr) \ - do { \ - (void)(addr); \ - } while (false) -#define COMMON_SYSCALL_FD_CLOSE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_FD_ACQUIRE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_FD_RELEASE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_PRE_FORK() \ - do { \ - } while (false) -#define COMMON_SYSCALL_POST_FORK(res) \ - do { \ - (void)(res); \ - } while (false) - -#include "sanitizer_common/sanitizer_common_syscalls.inc" -#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" - -//===----------------------------------------------------------------------===// -// Custom interceptors -//===----------------------------------------------------------------------===// - -// TODO(bruening): move more of these to the common interception pool as they -// are shared with tsan and asan. -// While our other files match LLVM style, here we match sanitizer style as we -// expect to move these to the common pool. - -INTERCEPTOR(char *, strcpy, char *dst, const char *src) { // NOLINT - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strcpy, dst, src); - uptr srclen = internal_strlen(src); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, srclen + 1); - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclen + 1); - return REAL(strcpy)(dst, src); // NOLINT -} - -INTERCEPTOR(char *, strncpy, char *dst, char *src, uptr n) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strncpy, dst, src, n); - uptr srclen = internal_strnlen(src, n); - uptr copied_size = srclen + 1 > n ? n : srclen + 1; - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, copied_size); - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, copied_size); - return REAL(strncpy)(dst, src, n); -} - -INTERCEPTOR(int, open, const char *name, int flags, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, open, name, flags, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(open)(name, flags, mode); -} - -#if SANITIZER_LINUX -INTERCEPTOR(int, open64, const char *name, int flags, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, open64, name, flags, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(open64)(name, flags, mode); -} -#define ESAN_MAYBE_INTERCEPT_OPEN64 INTERCEPT_FUNCTION(open64) -#else -#define ESAN_MAYBE_INTERCEPT_OPEN64 -#endif - -INTERCEPTOR(int, creat, const char *name, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, creat, name, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(creat)(name, mode); -} - -#if SANITIZER_LINUX -INTERCEPTOR(int, creat64, const char *name, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, creat64, name, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(creat64)(name, mode); -} -#define ESAN_MAYBE_INTERCEPT_CREAT64 INTERCEPT_FUNCTION(creat64) -#else -#define ESAN_MAYBE_INTERCEPT_CREAT64 -#endif - -INTERCEPTOR(int, unlink, char *path) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, unlink, path); - COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); - return REAL(unlink)(path); -} - -INTERCEPTOR(int, rmdir, char *path) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, rmdir, path); - COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); - return REAL(rmdir)(path); -} - -//===----------------------------------------------------------------------===// -// Signal-related interceptors -//===----------------------------------------------------------------------===// - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -typedef void (*signal_handler_t)(int); -INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, signal, signum, handler); - signal_handler_t result; - if (!processSignal(signum, handler, &result)) - return result; - else - return REAL(signal)(signum, handler); -} -#define ESAN_MAYBE_INTERCEPT_SIGNAL INTERCEPT_FUNCTION(signal) -#else -#error Platform not supported -#define ESAN_MAYBE_INTERCEPT_SIGNAL -#endif - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) -INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, sigaction, signum, act, oldact); - if (!processSigaction(signum, act, oldact)) - return 0; - else - return REAL(sigaction)(signum, act, oldact); -} - -// This is required to properly use internal_sigaction. -namespace __sanitizer { -int real_sigaction(int signum, const void *act, void *oldact) { - if (REAL(sigaction) == nullptr) { - // With an instrumented allocator, this is called during interceptor init - // and we need a raw syscall solution. -#if SANITIZER_LINUX - return internal_sigaction_syscall(signum, act, oldact); -#else - return internal_sigaction(signum, act, oldact); -#endif - } - return REAL(sigaction)(signum, (const struct sigaction *)act, - (struct sigaction *)oldact); -} -} // namespace __sanitizer - -#define ESAN_MAYBE_INTERCEPT_SIGACTION INTERCEPT_FUNCTION(sigaction) -#else -#error Platform not supported -#define ESAN_MAYBE_INTERCEPT_SIGACTION -#endif - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, - __sanitizer_sigset_t *oldset) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset); - int res = 0; - if (processSigprocmask(how, set, oldset)) - res = REAL(sigprocmask)(how, set, oldset); - if (!res && oldset) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); - return res; -} -#define ESAN_MAYBE_INTERCEPT_SIGPROCMASK INTERCEPT_FUNCTION(sigprocmask) -#else -#define ESAN_MAYBE_INTERCEPT_SIGPROCMASK -#endif - -#if !SANITIZER_WINDOWS -INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, - __sanitizer_sigset_t *oldset) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, pthread_sigmask, how, set, oldset); - int res = 0; - if (processSigprocmask(how, set, oldset)) - res = REAL(sigprocmask)(how, set, oldset); - if (!res && oldset) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); - return res; -} -#define ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK INTERCEPT_FUNCTION(pthread_sigmask) -#else -#define ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK -#endif - -//===----------------------------------------------------------------------===// -// Malloc interceptors -//===----------------------------------------------------------------------===// - -static const uptr early_alloc_buf_size = 4096; -static uptr allocated_bytes; -static char early_alloc_buf[early_alloc_buf_size]; - -static bool isInEarlyAllocBuf(const void *ptr) { - return ((uptr)ptr >= (uptr)early_alloc_buf && - ((uptr)ptr - (uptr)early_alloc_buf) < sizeof(early_alloc_buf)); -} - -static void *handleEarlyAlloc(uptr size) { - // If esan is initialized during an interceptor (which happens with some - // tcmalloc implementations that call pthread_mutex_lock), the call from - // dlsym to calloc will deadlock. - // dlsym may also call malloc before REAL(malloc) is retrieved from dlsym. - // We work around it by using a static buffer for the early malloc/calloc - // requests. - // This solution will also allow us to deliberately intercept malloc & family - // in the future (to perform tool actions on each allocation, without - // replacing the allocator), as it also solves the problem of intercepting - // calloc when it will itself be called before its REAL pointer is - // initialized. - // We do not handle multiple threads here. This only happens at process init - // time, and while it's possible for a shared library to create early threads - // that race here, we consider that to be a corner case extreme enough that - // it's not worth the effort to handle. - void *mem = (void *)&early_alloc_buf[allocated_bytes]; - allocated_bytes += size; - CHECK_LT(allocated_bytes, early_alloc_buf_size); - return mem; -} - -INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (EsanDuringInit && REAL(calloc) == nullptr) - return handleEarlyAlloc(size * n); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, calloc, size, n); - void *res = REAL(calloc)(size, n); - // The memory is zeroed and thus is all written. - COMMON_INTERCEPTOR_WRITE_RANGE(nullptr, (uptr)res, size * n); - return res; -} - -INTERCEPTOR(void*, malloc, uptr size) { - if (EsanDuringInit && REAL(malloc) == nullptr) - return handleEarlyAlloc(size); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, malloc, size); - return REAL(malloc)(size); -} - -INTERCEPTOR(void, free, void *p) { - void *ctx; - // There are only a few early allocation requests, so we simply skip the free. - if (isInEarlyAllocBuf(p)) - return; - COMMON_INTERCEPTOR_ENTER(ctx, free, p); - REAL(free)(p); -} - -namespace __esan { - -void initializeInterceptors() { - InitializeCommonInterceptors(); - - INTERCEPT_FUNCTION(strcpy); // NOLINT - INTERCEPT_FUNCTION(strncpy); - - INTERCEPT_FUNCTION(open); - ESAN_MAYBE_INTERCEPT_OPEN64; - INTERCEPT_FUNCTION(creat); - ESAN_MAYBE_INTERCEPT_CREAT64; - INTERCEPT_FUNCTION(unlink); - INTERCEPT_FUNCTION(rmdir); - - ESAN_MAYBE_INTERCEPT_SIGNAL; - ESAN_MAYBE_INTERCEPT_SIGACTION; - ESAN_MAYBE_INTERCEPT_SIGPROCMASK; - ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK; - - INTERCEPT_FUNCTION(calloc); - INTERCEPT_FUNCTION(malloc); - INTERCEPT_FUNCTION(free); - - // TODO(bruening): intercept routines that other sanitizers intercept that - // are not in the common pool or here yet, ideally by adding to the common - // pool. Examples include wcslen and bcopy. - - // TODO(bruening): there are many more libc routines that read or write data - // structures that no sanitizer is intercepting: sigaction, strtol, etc. -} - -} // namespace __esan diff --git a/lib/esan/esan_interface.cpp b/lib/esan/esan_interface.cpp deleted file mode 100644 index 43b3dff86..000000000 --- a/lib/esan/esan_interface.cpp +++ /dev/null @@ -1,122 +0,0 @@ -//===-- esan_interface.cpp ------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -//===----------------------------------------------------------------------===// - -#include "esan_interface_internal.h" -#include "esan.h" -#include "sanitizer_common/sanitizer_internal_defs.h" - -using namespace __esan; // NOLINT - -void __esan_init(ToolType Tool, void *Ptr) { - if (Tool != __esan_which_tool) { - Printf("ERROR: tool mismatch: %d vs %d\n", Tool, __esan_which_tool); - Die(); - } - initializeLibrary(Tool); - processCompilationUnitInit(Ptr); -} - -void __esan_exit(void *Ptr) { - processCompilationUnitExit(Ptr); -} - -void __esan_aligned_load1(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 1, false); -} - -void __esan_aligned_load2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, false); -} - -void __esan_aligned_load4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, false); -} - -void __esan_aligned_load8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, false); -} - -void __esan_aligned_load16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, false); -} - -void __esan_aligned_store1(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 1, true); -} - -void __esan_aligned_store2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, true); -} - -void __esan_aligned_store4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, true); -} - -void __esan_aligned_store8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, true); -} - -void __esan_aligned_store16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, true); -} - -void __esan_unaligned_load2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, false); -} - -void __esan_unaligned_load4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, false); -} - -void __esan_unaligned_load8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, false); -} - -void __esan_unaligned_load16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, false); -} - -void __esan_unaligned_store2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, true); -} - -void __esan_unaligned_store4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, true); -} - -void __esan_unaligned_store8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, true); -} - -void __esan_unaligned_store16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, true); -} - -void __esan_unaligned_loadN(void *Addr, uptr Size) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, Size, false); -} - -void __esan_unaligned_storeN(void *Addr, uptr Size) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, Size, true); -} - -// Public interface: -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE void __esan_report() { - reportResults(); -} - -SANITIZER_INTERFACE_ATTRIBUTE unsigned int __esan_get_sample_count() { - return getSampleCount(); -} -} // extern "C" diff --git a/lib/esan/esan_interface_internal.h b/lib/esan/esan_interface_internal.h deleted file mode 100644 index df51aa609..000000000 --- a/lib/esan/esan_interface_internal.h +++ /dev/null @@ -1,83 +0,0 @@ -//===-- esan_interface_internal.h -------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Calls to the functions declared in this header will be inserted by -// the instrumentation module. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_INTERFACE_INTERNAL_H -#define ESAN_INTERFACE_INTERNAL_H - -#include <sanitizer_common/sanitizer_internal_defs.h> - -// This header should NOT include any other headers. -// All functions in this header are extern "C" and start with __esan_. - -using __sanitizer::uptr; -using __sanitizer::u32; - -extern "C" { - -// This should be kept consistent with LLVM's EfficiencySanitizerOptions. -// The value is passed as a 32-bit integer by the compiler. -typedef enum Type : u32 { - ESAN_None = 0, - ESAN_CacheFrag, - ESAN_WorkingSet, - ESAN_Max, -} ToolType; - -// To handle interceptors that invoke instrumented code prior to -// __esan_init() being called, the instrumentation module creates this -// global variable specifying the tool. -extern ToolType __esan_which_tool; - -// This function should be called at the very beginning of the process, -// before any instrumented code is executed and before any call to malloc. -SANITIZER_INTERFACE_ATTRIBUTE void __esan_init(ToolType Tool, void *Ptr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_exit(void *Ptr); - -// The instrumentation module will insert a call to one of these routines prior -// to each load and store instruction for which we do not have "fastpath" -// inlined instrumentation. These calls constitute the "slowpath" for our -// tools. We have separate routines for each type of memory access to enable -// targeted optimization. -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load1(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store1(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store16(void *Addr); - -// These cover unusually-sized accesses. -SANITIZER_INTERFACE_ATTRIBUTE -void __esan_unaligned_loadN(void *Addr, uptr Size); -SANITIZER_INTERFACE_ATTRIBUTE -void __esan_unaligned_storeN(void *Addr, uptr Size); - -} // extern "C" - -#endif // ESAN_INTERFACE_INTERNAL_H diff --git a/lib/esan/esan_linux.cpp b/lib/esan/esan_linux.cpp deleted file mode 100644 index 014205ce0..000000000 --- a/lib/esan/esan_linux.cpp +++ /dev/null @@ -1,83 +0,0 @@ -//===-- esan.cpp ----------------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Linux-specific code for the Esan run-time. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX - -#include "esan.h" -#include "esan_shadow.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include <sys/mman.h> -#include <errno.h> - -namespace __esan { - -void verifyAddressSpace() { -#if SANITIZER_LINUX && (defined(__x86_64__) || SANITIZER_MIPS64) - // The kernel determines its mmap base from the stack size limit. - // Our Linux 64-bit shadow mapping assumes the stack limit is less than a - // terabyte, which keeps the mmap region above 0x7e00'. - uptr StackLimit = GetStackSizeLimitInBytes(); - if (StackSizeIsUnlimited() || StackLimit > MaxStackSize) { - VReport(1, "The stack size limit is beyond the maximum supported.\n" - "Re-execing with a stack size below 1TB.\n"); - SetStackSizeLimitInBytes(MaxStackSize); - ReExec(); - } -#endif -} - -static bool liesWithinSingleAppRegion(uptr Start, SIZE_T Size) { - uptr AppStart, AppEnd; - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - if (Start >= AppStart && Start + Size - 1 <= AppEnd) { - return true; - } - } - return false; -} - -bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags) { - if (*Addr) { - if (!liesWithinSingleAppRegion((uptr)*Addr, Size)) { - VPrintf(1, "mmap conflict: [%p-%p) is not in an app region\n", - *Addr, (uptr)*Addr + Size); - if (Flags & MAP_FIXED) { - errno = EINVAL; - return false; - } else { - *Addr = 0; - } - } - } - return true; -} - -uptr checkMmapResult(uptr Addr, SIZE_T Size) { - if ((void *)Addr == MAP_FAILED) - return Addr; - if (!liesWithinSingleAppRegion(Addr, Size)) { - // FIXME: attempt to dynamically add this as an app region if it - // fits our shadow criteria. - // We could also try to remap somewhere else. - Printf("ERROR: unsupported mapping at [%p-%p)\n", Addr, Addr+Size); - Die(); - } - return Addr; -} - -} // namespace __esan - -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/esan/esan_shadow.h b/lib/esan/esan_shadow.h deleted file mode 100644 index b76a9ccbd..000000000 --- a/lib/esan/esan_shadow.h +++ /dev/null @@ -1,292 +0,0 @@ -//===-- esan_shadow.h -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Shadow memory mappings for the esan run-time. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_SHADOW_H -#define ESAN_SHADOW_H - -#include "esan.h" -#include <sanitizer_common/sanitizer_platform.h> - -#if SANITIZER_WORDSIZE != 64 -#error Only 64-bit is supported -#endif - -namespace __esan { - -struct ApplicationRegion { - uptr Start; - uptr End; - bool ShadowMergedWithPrev; -}; - -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && defined(__x86_64__) -// Linux x86_64 -// -// Application memory falls into these 5 regions (ignoring the corner case -// of PIE with a non-zero PT_LOAD base): -// -// [0x00000000'00000000, 0x00000100'00000000) non-PIE + heap -// [0x00005500'00000000, 0x00005700'00000000) PIE -// [0x00007e00'00000000, 0x00007fff'ff600000) libraries + stack, part 1 -// [0x00007fff'ff601000, 0x00008000'00000000) libraries + stack, part 2 -// [0xffffffff'ff600000, 0xffffffff'ff601000) vsyscall -// -// Although we can ignore the vsyscall for the most part as there are few data -// references there (other sanitizers ignore it), we enforce a gap inside the -// library region to distinguish the vsyscall's shadow, considering this gap to -// be an invalid app region. -// We disallow application memory outside of those 5 regions. -// Our regions assume that the stack rlimit is less than a terabyte (otherwise -// the Linux kernel's default mmap region drops below 0x7e00'), which we enforce -// at init time (we can support larger and unlimited sizes for shadow -// scaledowns, but it is difficult for 1:1 mappings). -// -// Our shadow memory is scaled from a 1:1 mapping and supports a scale -// specified at library initialization time that can be any power-of-2 -// scaledown (1x, 2x, 4x, 8x, 16x, etc.). -// -// We model our shadow memory after Umbra, a library used by the Dr. Memory -// tool: https://github.com/DynamoRIO/drmemory/blob/master/umbra/umbra_x64.c. -// We use Umbra's scheme as it was designed to support different -// offsets, it supports two different shadow mappings (which we may want to -// use for future tools), and it ensures that the shadow of a shadow will -// not overlap either shadow memory or application memory. -// -// This formula translates from application memory to shadow memory: -// -// shadow(app) = ((app & 0x00000fff'ffffffff) + offset) >> scale -// -// Where the offset for 1:1 is 0x00001300'00000000. For other scales, the -// offset is shifted left by the scale, except for scales of 1 and 2 where -// it must be tweaked in order to pass the double-shadow test -// (see the "shadow(shadow)" comments below): -// scale == 0: 0x00001300'000000000 -// scale == 1: 0x00002200'000000000 -// scale == 2: 0x00004400'000000000 -// scale >= 3: (0x00001300'000000000 << scale) -// -// Do not pass in the open-ended end value to the formula as it will fail. -// -// The resulting shadow memory regions for a 0 scaling are: -// -// [0x00001300'00000000, 0x00001400'00000000) -// [0x00001800'00000000, 0x00001a00'00000000) -// [0x00002100'00000000, 0x000022ff'ff600000) -// [0x000022ff'ff601000, 0x00002300'00000000) -// [0x000022ff'ff600000, 0x000022ff'ff601000] -// -// We also want to ensure that a wild access by the application into the shadow -// regions will not corrupt our own shadow memory. shadow(shadow) ends up -// disjoint from shadow(app): -// -// [0x00001600'00000000, 0x00001700'00000000) -// [0x00001b00'00000000, 0x00001d00'00000000) -// [0x00001400'00000000, 0x000015ff'ff600000] -// [0x000015ff'ff601000, 0x00001600'00000000] -// [0x000015ff'ff600000, 0x000015ff'ff601000] - -static const struct ApplicationRegion AppRegions[] = { - {0x0000000000000000ull, 0x0000010000000000u, false}, - {0x0000550000000000u, 0x0000570000000000u, false}, - // We make one shadow mapping to hold the shadow regions for all 3 of these - // app regions, as the mappings interleave, and the gap between the 3rd and - // 4th scales down below a page. - {0x00007e0000000000u, 0x00007fffff600000u, false}, - {0x00007fffff601000u, 0x0000800000000000u, true}, - {0xffffffffff600000u, 0xffffffffff601000u, true}, -}; - -#elif SANITIZER_LINUX && SANITIZER_MIPS64 - -// Application memory falls into these 3 regions -// -// [0x00000001'00000000, 0x00000002'00000000) non-PIE + heap -// [0x000000aa'00000000, 0x000000ab'00000000) PIE -// [0x000000ff'00000000, 0x000000ff'ffffffff) libraries + stack -// -// This formula translates from application memory to shadow memory: -// -// shadow(app) = ((app & 0x00000f'ffffffff) + offset) >> scale -// -// Where the offset for 1:1 is 0x000013'00000000. For other scales, the -// offset is shifted left by the scale, except for scales of 1 and 2 where -// it must be tweaked in order to pass the double-shadow test -// (see the "shadow(shadow)" comments below): -// scale == 0: 0x000013'00000000 -// scale == 1: 0x000022'00000000 -// scale == 2: 0x000044'00000000 -// scale >= 3: (0x000013'00000000 << scale) -// -// The resulting shadow memory regions for a 0 scaling are: -// -// [0x00000014'00000000, 0x00000015'00000000) -// [0x0000001d'00000000, 0x0000001e'00000000) -// [0x00000022'00000000, 0x00000022'ffffffff) -// -// We also want to ensure that a wild access by the application into the shadow -// regions will not corrupt our own shadow memory. shadow(shadow) ends up -// disjoint from shadow(app): -// -// [0x00000017'00000000, 0x00000018'00000000) -// [0x00000020'00000000, 0x00000021'00000000) -// [0x00000015'00000000, 0x00000015'ffffffff] - -static const struct ApplicationRegion AppRegions[] = { - {0x0100000000u, 0x0200000000u, false}, - {0xaa00000000u, 0xab00000000u, false}, - {0xff00000000u, 0xffffffffffu, false}, -}; - -#else -#error Platform not supported -#endif - -static const u32 NumAppRegions = sizeof(AppRegions)/sizeof(AppRegions[0]); - -// See the comment above: we do not currently support a stack size rlimit -// equal to or larger than 1TB. -static const uptr MaxStackSize = (1ULL << 40) - 4096; - -class ShadowMapping { -public: - - // The scale and offset vary by tool. - uptr Scale; - uptr Offset; - - // TODO(sagar.thakur): Try to hardcode the mask as done in the compiler - // instrumentation to reduce the runtime cost of appToShadow. - struct ShadowMemoryMask40 { - static const uptr Mask = 0x0000000fffffffffu; - }; - - struct ShadowMemoryMask47 { - static const uptr Mask = 0x00000fffffffffffu; - }; - - void initialize(uptr ShadowScale) { - - const uptr OffsetArray40[3] = { - 0x0000001300000000u, - 0x0000002200000000u, - 0x0000004400000000u, - }; - - const uptr OffsetArray47[3] = { - 0x0000130000000000u, - 0x0000220000000000u, - 0x0000440000000000u, - }; - - Scale = ShadowScale; - switch (VmaSize) { - case 40: { - if (Scale <= 2) - Offset = OffsetArray40[Scale]; - else - Offset = OffsetArray40[0] << Scale; - } - break; - case 47: { - if (Scale <= 2) - Offset = OffsetArray47[Scale]; - else - Offset = OffsetArray47[0] << Scale; - } - break; - default: { - Printf("ERROR: %d-bit virtual memory address size not supported\n", VmaSize); - Die(); - } - } - } -}; -extern ShadowMapping Mapping; - -static inline bool getAppRegion(u32 i, uptr *Start, uptr *End) { - if (i >= NumAppRegions) - return false; - *Start = AppRegions[i].Start; - *End = AppRegions[i].End; - return true; -} - -ALWAYS_INLINE -bool isAppMem(uptr Mem) { - for (u32 i = 0; i < NumAppRegions; ++i) { - if (Mem >= AppRegions[i].Start && Mem < AppRegions[i].End) - return true; - } - return false; -} - -template<typename Params> -uptr appToShadowImpl(uptr App) { - return (((App & Params::Mask) + Mapping.Offset) >> Mapping.Scale); -} - -ALWAYS_INLINE -uptr appToShadow(uptr App) { - switch (VmaSize) { - case 40: return appToShadowImpl<ShadowMapping::ShadowMemoryMask40>(App); - case 47: return appToShadowImpl<ShadowMapping::ShadowMemoryMask47>(App); - default: { - Printf("ERROR: %d-bit virtual memory address size not supported\n", VmaSize); - Die(); - } - } -} - -static inline bool getShadowRegion(u32 i, uptr *Start, uptr *End) { - if (i >= NumAppRegions) - return false; - u32 UnmergedShadowCount = 0; - u32 AppIdx; - for (AppIdx = 0; AppIdx < NumAppRegions; ++AppIdx) { - if (!AppRegions[AppIdx].ShadowMergedWithPrev) { - if (UnmergedShadowCount == i) - break; - UnmergedShadowCount++; - } - } - if (AppIdx >= NumAppRegions || UnmergedShadowCount != i) - return false; - *Start = appToShadow(AppRegions[AppIdx].Start); - // The formula fails for the end itself. - *End = appToShadow(AppRegions[AppIdx].End - 1) + 1; - // Merge with adjacent shadow regions: - for (++AppIdx; AppIdx < NumAppRegions; ++AppIdx) { - if (!AppRegions[AppIdx].ShadowMergedWithPrev) - break; - *Start = Min(*Start, appToShadow(AppRegions[AppIdx].Start)); - *End = Max(*End, appToShadow(AppRegions[AppIdx].End - 1) + 1); - } - return true; -} - -ALWAYS_INLINE -bool isShadowMem(uptr Mem) { - // We assume this is not used on any critical performance path and so there's - // no need to hardcode the mapping results. - for (uptr i = 0; i < NumAppRegions; ++i) { - if (Mem >= appToShadow(AppRegions[i].Start) && - Mem < appToShadow(AppRegions[i].End - 1) + 1) - return true; - } - return false; -} - -} // namespace __esan - -#endif /* ESAN_SHADOW_H */ diff --git a/lib/esan/esan_sideline.h b/lib/esan/esan_sideline.h deleted file mode 100644 index 74551fbde..000000000 --- a/lib/esan/esan_sideline.h +++ /dev/null @@ -1,64 +0,0 @@ -//===-- esan_sideline.h -----------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan sideline thread support. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_SIDELINE_H -#define ESAN_SIDELINE_H - -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_platform_limits_freebsd.h" -#include "sanitizer_common/sanitizer_platform_limits_posix.h" - -namespace __esan { - -typedef void (*SidelineFunc)(void *Arg); - -// Currently only one sideline thread is supported. -// It calls the SidelineFunc passed to launchThread once on each sample at the -// given frequency in real time (i.e., wall clock time). -class SidelineThread { -public: - // We cannot initialize any fields in the constructor as it will be called - // *after* launchThread for a static instance, as esan.module_ctor is called - // before static initializers. - SidelineThread() {} - ~SidelineThread() {} - - // To simplify declaration in sanitizer code where we want to avoid - // heap allocations, the constructor and destructor do nothing and - // launchThread and joinThread do the real work. - // They should each be called just once. - bool launchThread(SidelineFunc takeSample, void *Arg, u32 FreqMilliSec); - bool joinThread(); - - // Must be called from the sideline thread itself. - bool adjustTimer(u32 FreqMilliSec); - -private: - static int runSideline(void *Arg); - static void registerSignal(int SigNum); - static void handleSidelineSignal(int SigNum, __sanitizer_siginfo *SigInfo, - void *Ctx); - - char *Stack; - SidelineFunc sampleFunc; - void *FuncArg; - u32 Freq; - uptr SidelineId; - atomic_uintptr_t SidelineExit; -}; - -} // namespace __esan - -#endif // ESAN_SIDELINE_H diff --git a/lib/esan/esan_sideline_bsd.cpp b/lib/esan/esan_sideline_bsd.cpp deleted file mode 100644 index 3134d3776..000000000 --- a/lib/esan/esan_sideline_bsd.cpp +++ /dev/null @@ -1,35 +0,0 @@ -//===-- esan_sideline_bsd.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Support for a separate or "sideline" tool thread on FreeBSD. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD - -#include "esan_sideline.h" - -namespace __esan { - -static SidelineThread *TheThread; - -bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, - u32 FreqMilliSec) { - return true; -} - -bool SidelineThread::joinThread() { - return true; -} - -} // namespace __esan - -#endif // SANITIZER_FREEBSD diff --git a/lib/esan/esan_sideline_linux.cpp b/lib/esan/esan_sideline_linux.cpp deleted file mode 100644 index 2de25fba7..000000000 --- a/lib/esan/esan_sideline_linux.cpp +++ /dev/null @@ -1,178 +0,0 @@ -//===-- esan_sideline_linux.cpp ---------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Support for a separate or "sideline" tool thread on Linux. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX - -#include "esan_sideline.h" -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_linux.h" -#include <errno.h> -#include <sched.h> -#include <sys/prctl.h> -#include <sys/signal.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/wait.h> - -namespace __esan { - -static const int SigAltStackSize = 4*1024; -static const int SidelineStackSize = 4*1024; -static const uptr SidelineIdUninitialized = 1; - -// FIXME: we'll need some kind of TLS (can we trust that a pthread key will -// work in our non-POSIX thread?) to access our data in our signal handler -// with multiple sideline threads. For now we assume there is only one -// sideline thread and we use a dirty solution of a global var. -static SidelineThread *TheThread; - -// We aren't passing SA_NODEFER so the same signal is blocked while here. -void SidelineThread::handleSidelineSignal(int SigNum, - __sanitizer_siginfo *SigInfo, - void *Ctx) { - VPrintf(3, "Sideline signal %d\n", SigNum); - CHECK_EQ(SigNum, SIGALRM); - // See above about needing TLS to avoid this global var. - SidelineThread *Thread = TheThread; - if (atomic_load(&Thread->SidelineExit, memory_order_relaxed) != 0) - return; - Thread->sampleFunc(Thread->FuncArg); -} - -void SidelineThread::registerSignal(int SigNum) { - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = handleSidelineSignal; - // We do not pass SA_NODEFER as we want to block the same signal. - SigAct.sa_flags = SA_ONSTACK | SA_SIGINFO; - int Res = internal_sigaction(SigNum, &SigAct, nullptr); - CHECK_EQ(Res, 0); -} - -int SidelineThread::runSideline(void *Arg) { - VPrintf(1, "Sideline thread starting\n"); - SidelineThread *Thread = static_cast<SidelineThread*>(Arg); - - // If the parent dies, we want to exit also. - internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); - - // Set up a signal handler on an alternate stack for safety. - InternalMmapVector<char> StackMap(SigAltStackSize); - stack_t SigAltStack; - SigAltStack.ss_sp = StackMap.data(); - SigAltStack.ss_size = SigAltStackSize; - SigAltStack.ss_flags = 0; - internal_sigaltstack(&SigAltStack, nullptr); - - // We inherit the signal mask from the app thread. In case - // we weren't created at init time, we ensure the mask is empty. - __sanitizer_sigset_t SigSet; - internal_sigfillset(&SigSet); - int Res = internal_sigprocmask(SIG_UNBLOCK, &SigSet, nullptr); - CHECK_EQ(Res, 0); - - registerSignal(SIGALRM); - - bool TimerSuccess = Thread->adjustTimer(Thread->Freq); - CHECK(TimerSuccess); - - // We loop, doing nothing but handling itimer signals. - while (atomic_load(&TheThread->SidelineExit, memory_order_relaxed) == 0) - sched_yield(); - - if (!Thread->adjustTimer(0)) - VPrintf(1, "Failed to disable timer\n"); - - VPrintf(1, "Sideline thread exiting\n"); - return 0; -} - -bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, - u32 FreqMilliSec) { - // This can only be called once. However, we can't clear a field in - // the constructor and check for that here as the constructor for - // a static instance is called *after* our module_ctor and thus after - // this routine! Thus we rely on the TheThread check below. - CHECK(TheThread == nullptr); // Only one sideline thread is supported. - TheThread = this; - sampleFunc = takeSample; - FuncArg = Arg; - Freq = FreqMilliSec; - atomic_store(&SidelineExit, 0, memory_order_relaxed); - - // We do without a guard page. - Stack = static_cast<char*>(MmapOrDie(SidelineStackSize, "SidelineStack")); - // We need to handle the return value from internal_clone() not having been - // assigned yet (for our CHECK in adjustTimer()) so we ensure this has a - // sentinel value. - SidelineId = SidelineIdUninitialized; - // By omitting CLONE_THREAD, the child is in its own thread group and will not - // receive any of the application's signals. - SidelineId = internal_clone( - runSideline, Stack + SidelineStackSize, - CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED, - this, nullptr /* parent_tidptr */, - nullptr /* newtls */, nullptr /* child_tidptr */); - int ErrCode; - if (internal_iserror(SidelineId, &ErrCode)) { - Printf("FATAL: EfficiencySanitizer failed to spawn a thread (code %d).\n", - ErrCode); - Die(); - return false; // Not reached. - } - return true; -} - -bool SidelineThread::joinThread() { - VPrintf(1, "Joining sideline thread\n"); - bool Res = true; - atomic_store(&SidelineExit, 1, memory_order_relaxed); - while (true) { - uptr Status = internal_waitpid(SidelineId, nullptr, __WALL); - int ErrCode; - if (!internal_iserror(Status, &ErrCode)) - break; - if (ErrCode == EINTR) - continue; - VPrintf(1, "Failed to join sideline thread (errno %d)\n", ErrCode); - Res = false; - break; - } - UnmapOrDie(Stack, SidelineStackSize); - return Res; -} - -// Must be called from the sideline thread itself. -bool SidelineThread::adjustTimer(u32 FreqMilliSec) { - // The return value of internal_clone() may not have been assigned yet: - CHECK(internal_getpid() == SidelineId || - SidelineId == SidelineIdUninitialized); - Freq = FreqMilliSec; - struct itimerval TimerVal; - TimerVal.it_interval.tv_sec = (time_t) Freq / 1000; - TimerVal.it_interval.tv_usec = (time_t) (Freq % 1000) * 1000; - TimerVal.it_value.tv_sec = (time_t) Freq / 1000; - TimerVal.it_value.tv_usec = (time_t) (Freq % 1000) * 1000; - // As we're in a different thread group, we cannot use either - // ITIMER_PROF or ITIMER_VIRTUAL without taking up scheduled - // time ourselves: thus we must use real time. - int Res = setitimer(ITIMER_REAL, &TimerVal, nullptr); - return (Res == 0); -} - -} // namespace __esan - -#endif // SANITIZER_LINUX diff --git a/lib/esan/working_set.cpp b/lib/esan/working_set.cpp deleted file mode 100644 index e56902c8f..000000000 --- a/lib/esan/working_set.cpp +++ /dev/null @@ -1,280 +0,0 @@ -//===-- working_set.cpp ---------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// This file contains working-set-specific code. -//===----------------------------------------------------------------------===// - -#include "working_set.h" -#include "esan.h" -#include "esan_circular_buffer.h" -#include "esan_flags.h" -#include "esan_shadow.h" -#include "esan_sideline.h" -#include "sanitizer_common/sanitizer_procmaps.h" - -// We shadow every cache line of app memory with one shadow byte. -// - The highest bit of each shadow byte indicates whether the corresponding -// cache line has ever been accessed. -// - The lowest bit of each shadow byte indicates whether the corresponding -// cache line was accessed since the last sample. -// - The other bits are used for working set snapshots at successively -// lower frequencies, each bit to the left from the lowest bit stepping -// down the frequency by 2 to the power of getFlags()->snapshot_step. -// Thus we have something like this: -// Bit 0: Since last sample -// Bit 1: Since last 2^2 samples -// Bit 2: Since last 2^4 samples -// Bit 3: ... -// Bit 7: Ever accessed. -// We live with races in accessing each shadow byte. -typedef unsigned char byte; - -namespace __esan { - -// Our shadow memory assumes that the line size is 64. -static const u32 CacheLineSize = 64; - -// See the shadow byte layout description above. -static const u32 TotalWorkingSetBitIdx = 7; -// We accumulate to the left until we hit this bit. -// We don't need to accumulate to the final bit as it's set on each ref -// by the compiler instrumentation. -static const u32 MaxAccumBitIdx = 6; -static const u32 CurWorkingSetBitIdx = 0; -static const byte ShadowAccessedVal = - (1 << TotalWorkingSetBitIdx) | (1 << CurWorkingSetBitIdx); - -static SidelineThread Thread; -// If we use real-time-based timer samples this won't overflow in any realistic -// scenario, but if we switch to some other unit (such as memory accesses) we -// may want to consider a 64-bit int. -static u32 SnapshotNum; - -// We store the wset size for each of 8 different sampling frequencies. -static const u32 NumFreq = 8; // One for each bit of our shadow bytes. -// We cannot use static objects as the global destructor is called -// prior to our finalize routine. -// These are each circular buffers, sized up front. -CircularBuffer<u32> SizePerFreq[NumFreq]; -// We cannot rely on static initializers (they may run too late) but -// we record the size here for clarity: -u32 CircularBufferSizes[NumFreq] = { - // These are each mmap-ed so our minimum is one page. - 32*1024, - 16*1024, - 8*1024, - 4*1024, - 4*1024, - 4*1024, - 4*1024, - 4*1024, -}; - -void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, - bool IsWrite) { - if (Size == 0) - return; - SIZE_T I = 0; - uptr LineSize = getFlags()->cache_line_size; - // As Addr+Size could overflow at the top of a 32-bit address space, - // we avoid the simpler formula that rounds the start and end. - SIZE_T NumLines = Size / LineSize + - // Add any extra at the start or end adding on an extra line: - (LineSize - 1 + Addr % LineSize + Size % LineSize) / LineSize; - byte *Shadow = (byte *)appToShadow(Addr); - // Write shadow bytes until we're word-aligned. - while (I < NumLines && (uptr)Shadow % 4 != 0) { - if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) - *Shadow |= ShadowAccessedVal; - ++Shadow; - ++I; - } - // Write whole shadow words at a time. - // Using a word-stride loop improves the runtime of a microbenchmark of - // memset calls by 10%. - u32 WordValue = ShadowAccessedVal | ShadowAccessedVal << 8 | - ShadowAccessedVal << 16 | ShadowAccessedVal << 24; - while (I + 4 <= NumLines) { - if ((*(u32*)Shadow & WordValue) != WordValue) - *(u32*)Shadow |= WordValue; - Shadow += 4; - I += 4; - } - // Write any trailing shadow bytes. - while (I < NumLines) { - if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) - *Shadow |= ShadowAccessedVal; - ++Shadow; - ++I; - } -} - -// This routine will word-align ShadowStart and ShadowEnd prior to scanning. -// It does *not* clear for BitIdx==TotalWorkingSetBitIdx, as that top bit -// measures the access during the entire execution and should never be cleared. -static u32 countAndClearShadowValues(u32 BitIdx, uptr ShadowStart, - uptr ShadowEnd) { - u32 WorkingSetSize = 0; - u32 ByteValue = 0x1 << BitIdx; - u32 WordValue = ByteValue | ByteValue << 8 | ByteValue << 16 | - ByteValue << 24; - // Get word aligned start. - ShadowStart = RoundDownTo(ShadowStart, sizeof(u32)); - bool Accum = getFlags()->record_snapshots && BitIdx < MaxAccumBitIdx; - // Do not clear the bit that measures access during the entire execution. - bool Clear = BitIdx < TotalWorkingSetBitIdx; - for (u32 *Ptr = (u32 *)ShadowStart; Ptr < (u32 *)ShadowEnd; ++Ptr) { - if ((*Ptr & WordValue) != 0) { - byte *BytePtr = (byte *)Ptr; - for (u32 j = 0; j < sizeof(u32); ++j) { - if (BytePtr[j] & ByteValue) { - ++WorkingSetSize; - if (Accum) { - // Accumulate to the lower-frequency bit to the left. - BytePtr[j] |= (ByteValue << 1); - } - } - } - if (Clear) { - // Clear this bit from every shadow byte. - *Ptr &= ~WordValue; - } - } - } - return WorkingSetSize; -} - -// Scan shadow memory to calculate the number of cache lines being accessed, -// i.e., the number of non-zero bits indexed by BitIdx in each shadow byte. -// We also clear the lowest bits (most recent working set snapshot). -// We do *not* clear for BitIdx==TotalWorkingSetBitIdx, as that top bit -// measures the access during the entire execution and should never be cleared. -static u32 computeWorkingSizeAndReset(u32 BitIdx) { - u32 WorkingSetSize = 0; - MemoryMappingLayout MemIter(true/*cache*/); - MemoryMappedSegment Segment; - while (MemIter.Next(&Segment)) { - VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", __FUNCTION__, - Segment.start, Segment.end, Segment.protection, - isAppMem(Segment.start), isShadowMem(Segment.start)); - if (isShadowMem(Segment.start) && Segment.IsWritable()) { - VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Segment.start, - Segment.end); - WorkingSetSize += - countAndClearShadowValues(BitIdx, Segment.start, Segment.end); - } - } - return WorkingSetSize; -} - -// This is invoked from a signal handler but in a sideline thread doing nothing -// else so it is a little less fragile than a typical signal handler. -static void takeSample(void *Arg) { - u32 BitIdx = CurWorkingSetBitIdx; - u32 Freq = 1; - ++SnapshotNum; // Simpler to skip 0 whose mod matches everything. - while (BitIdx <= MaxAccumBitIdx && (SnapshotNum % Freq) == 0) { - u32 NumLines = computeWorkingSizeAndReset(BitIdx); - VReport(1, "%s: snapshot #%5d bit %d freq %4d: %8u\n", SanitizerToolName, - SnapshotNum, BitIdx, Freq, NumLines); - SizePerFreq[BitIdx].push_back(NumLines); - Freq = Freq << getFlags()->snapshot_step; - BitIdx++; - } -} - -unsigned int getSampleCountWorkingSet() -{ - return SnapshotNum; -} - -// Initialization that must be done before any instrumented code is executed. -void initializeShadowWorkingSet() { - CHECK(getFlags()->cache_line_size == CacheLineSize); - registerMemoryFaultHandler(); -} - -void initializeWorkingSet() { - if (getFlags()->record_snapshots) { - for (u32 i = 0; i < NumFreq; ++i) - SizePerFreq[i].initialize(CircularBufferSizes[i]); - Thread.launchThread(takeSample, nullptr, getFlags()->sample_freq); - } -} - -static u32 getPeriodForPrinting(u32 MilliSec, const char *&Unit) { - if (MilliSec > 600000) { - Unit = "min"; - return MilliSec / 60000; - } else if (MilliSec > 10000) { - Unit = "sec"; - return MilliSec / 1000; - } else { - Unit = "ms"; - return MilliSec; - } -} - -static u32 getSizeForPrinting(u32 NumOfCachelines, const char *&Unit) { - // We need a constant to avoid software divide support: - static const u32 KilobyteCachelines = (0x1 << 10) / CacheLineSize; - static const u32 MegabyteCachelines = KilobyteCachelines << 10; - - if (NumOfCachelines > 10 * MegabyteCachelines) { - Unit = "MB"; - return NumOfCachelines / MegabyteCachelines; - } else if (NumOfCachelines > 10 * KilobyteCachelines) { - Unit = "KB"; - return NumOfCachelines / KilobyteCachelines; - } else { - Unit = "Bytes"; - return NumOfCachelines * CacheLineSize; - } -} - -void reportWorkingSet() { - const char *Unit; - if (getFlags()->record_snapshots) { - u32 Freq = 1; - Report(" Total number of samples: %u\n", SnapshotNum); - for (u32 i = 0; i < NumFreq; ++i) { - u32 Time = getPeriodForPrinting(getFlags()->sample_freq*Freq, Unit); - Report(" Samples array #%d at period %u %s\n", i, Time, Unit); - // FIXME: report whether we wrapped around and thus whether we - // have data on the whole run or just the last N samples. - for (u32 j = 0; j < SizePerFreq[i].size(); ++j) { - u32 Size = getSizeForPrinting(SizePerFreq[i][j], Unit); - Report("#%4d: %8u %s (%9u cache lines)\n", j, Size, Unit, - SizePerFreq[i][j]); - } - Freq = Freq << getFlags()->snapshot_step; - } - } - - // Get the working set size for the entire execution. - u32 NumOfCachelines = computeWorkingSizeAndReset(TotalWorkingSetBitIdx); - u32 Size = getSizeForPrinting(NumOfCachelines, Unit); - Report(" %s: the total working set size: %u %s (%u cache lines)\n", - SanitizerToolName, Size, Unit, NumOfCachelines); -} - -int finalizeWorkingSet() { - if (getFlags()->record_snapshots) - Thread.joinThread(); - reportWorkingSet(); - if (getFlags()->record_snapshots) { - for (u32 i = 0; i < NumFreq; ++i) - SizePerFreq[i].free(); - } - return 0; -} - -} // namespace __esan diff --git a/lib/esan/working_set.h b/lib/esan/working_set.h deleted file mode 100644 index 6a976c3f9..000000000 --- a/lib/esan/working_set.h +++ /dev/null @@ -1,40 +0,0 @@ -//===-- working_set.h -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Header for working-set-specific code. -//===----------------------------------------------------------------------===// - -#ifndef WORKING_SET_H -#define WORKING_SET_H - -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_internal_defs.h" - -namespace __esan { - -void initializeWorkingSet(); -void initializeShadowWorkingSet(); -int finalizeWorkingSet(); -void reportWorkingSet(); -unsigned int getSampleCountWorkingSet(); -void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, - bool IsWrite); - -// Platform-dependent. -void registerMemoryFaultHandler(); -bool processWorkingSetSignal(int SigNum, void (*Handler)(int), - void (**Result)(int)); -bool processWorkingSetSigaction(int SigNum, const void *Act, void *OldAct); -bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet); - -} // namespace __esan - -#endif // WORKING_SET_H diff --git a/lib/esan/working_set_posix.cpp b/lib/esan/working_set_posix.cpp deleted file mode 100644 index 5ec53b959..000000000 --- a/lib/esan/working_set_posix.cpp +++ /dev/null @@ -1,134 +0,0 @@ -//===-- working_set_posix.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// POSIX-specific working set tool code. -//===----------------------------------------------------------------------===// - -#include "working_set.h" -#include "esan_flags.h" -#include "esan_shadow.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_linux.h" -#include <signal.h> -#include <sys/mman.h> - -namespace __esan { - -// We only support regular POSIX threads with a single signal handler -// for the whole process == thread group. -// Thus we only need to store one app signal handler. -// FIXME: Store and use any alternate stack and signal flags set by -// the app. For now we just call the app handler from our handler. -static __sanitizer_sigaction AppSigAct; - -bool processWorkingSetSignal(int SigNum, void (*Handler)(int), - void (**Result)(int)) { - VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); - if (SigNum == SIGSEGV) { - *Result = AppSigAct.handler; - AppSigAct.sigaction = (decltype(AppSigAct.sigaction))Handler; - return false; // Skip real call. - } - return true; -} - -bool processWorkingSetSigaction(int SigNum, const void *ActVoid, - void *OldActVoid) { - VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); - if (SigNum == SIGSEGV) { - const struct sigaction *Act = (const struct sigaction *) ActVoid; - struct sigaction *OldAct = (struct sigaction *) OldActVoid; - if (OldAct) - internal_memcpy(OldAct, &AppSigAct, sizeof(OldAct)); - if (Act) - internal_memcpy(&AppSigAct, Act, sizeof(AppSigAct)); - return false; // Skip real call. - } - return true; -} - -bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet) { - VPrintf(2, "%s\n", __FUNCTION__); - // All we need to do is ensure that SIGSEGV is not blocked. - // FIXME: we are not fully transparent as we do not pretend that - // SIGSEGV is still blocked on app queries: that would require - // per-thread mask tracking. - if (Set && (How == SIG_BLOCK || How == SIG_SETMASK)) { - if (internal_sigismember((__sanitizer_sigset_t *)Set, SIGSEGV)) { - VPrintf(1, "%s: removing SIGSEGV from the blocked set\n", __FUNCTION__); - internal_sigdelset((__sanitizer_sigset_t *)Set, SIGSEGV); - } - } - return true; -} - -static void reinstateDefaultHandler(int SigNum) { - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = (decltype(SigAct.sigaction))SIG_DFL; - int Res = internal_sigaction(SigNum, &SigAct, nullptr); - CHECK(Res == 0); - VPrintf(1, "Unregistered for %d handler\n", SigNum); -} - -// If this is a shadow fault, we handle it here; otherwise, we pass it to the -// app to handle it just as the app would do without our tool in place. -static void handleMemoryFault(int SigNum, __sanitizer_siginfo *Info, - void *Ctx) { - if (SigNum == SIGSEGV) { - // We rely on si_addr being filled in (thus we do not support old kernels). - siginfo_t *SigInfo = (siginfo_t *)Info; - uptr Addr = (uptr)SigInfo->si_addr; - if (isShadowMem(Addr)) { - VPrintf(3, "Shadow fault @%p\n", Addr); - uptr PageSize = GetPageSizeCached(); - int Res = internal_mprotect((void *)RoundDownTo(Addr, PageSize), - PageSize, PROT_READ|PROT_WRITE); - CHECK(Res == 0); - } else if (AppSigAct.sigaction) { - // FIXME: For simplicity we ignore app options including its signal stack - // (we just use ours) and all the delivery flags. - AppSigAct.sigaction(SigNum, Info, Ctx); - } else { - // Crash instead of spinning with infinite faults. - reinstateDefaultHandler(SigNum); - } - } else - UNREACHABLE("signal not registered"); -} - -void registerMemoryFaultHandler() { - // We do not use an alternate signal stack, as doing so would require - // setting it up for each app thread. - // FIXME: This could result in problems with emulating the app's signal - // handling if the app relies on an alternate stack for SIGSEGV. - - // We require that SIGSEGV is not blocked. We use a sigprocmask - // interceptor to ensure that in the future. Here we ensure it for - // the current thread. We assume there are no other threads at this - // point during initialization, or that at least they do not block - // SIGSEGV. - __sanitizer_sigset_t SigSet; - internal_sigemptyset(&SigSet); - internal_sigprocmask(SIG_BLOCK, &SigSet, nullptr); - - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = handleMemoryFault; - // We want to handle nested signals b/c we need to handle a - // shadow fault in an app signal handler. - SigAct.sa_flags = SA_SIGINFO | SA_NODEFER; - int Res = internal_sigaction(SIGSEGV, &SigAct, &AppSigAct); - CHECK(Res == 0); - VPrintf(1, "Registered for SIGSEGV handler\n"); -} - -} // namespace __esan diff --git a/lib/fuzzer/CMakeLists.txt b/lib/fuzzer/CMakeLists.txt index caea9734f..852019dc1 100644 --- a/lib/fuzzer/CMakeLists.txt +++ b/lib/fuzzer/CMakeLists.txt @@ -6,6 +6,7 @@ set(LIBFUZZER_SOURCES FuzzerExtFunctionsWeak.cpp FuzzerExtFunctionsWindows.cpp FuzzerExtraCounters.cpp + FuzzerFork.cpp FuzzerIO.cpp FuzzerIOPosix.cpp FuzzerIOWindows.cpp @@ -13,9 +14,6 @@ set(LIBFUZZER_SOURCES FuzzerMerge.cpp FuzzerMutate.cpp FuzzerSHA1.cpp - FuzzerShmemFuchsia.cpp - FuzzerShmemPosix.cpp - FuzzerShmemWindows.cpp FuzzerTracePC.cpp FuzzerUtil.cpp FuzzerUtilDarwin.cpp @@ -35,6 +33,7 @@ set(LIBFUZZER_HEADERS FuzzerExtFunctions.def FuzzerExtFunctions.h FuzzerFlags.def + FuzzerFork.h FuzzerIO.h FuzzerInterface.h FuzzerInternal.h @@ -43,7 +42,6 @@ set(LIBFUZZER_HEADERS FuzzerOptions.h FuzzerRandom.h FuzzerSHA1.h - FuzzerShmem.h FuzzerTracePC.h FuzzerUtil.h FuzzerValueBitMap.h) @@ -57,7 +55,9 @@ CHECK_CXX_SOURCE_COMPILES(" set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) -if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) +if(OS_NAME MATCHES "Linux|Fuchsia" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer) # Remove -stdlib= which is unused when passing -nostdinc++. string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) @@ -71,12 +71,21 @@ if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage") list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters) endif() -if(NOT HAS_THREAD_LOCAL) - list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread) +if(MSVC) + # Silence warnings by turning off exceptions in MSVC headers and avoid an + # error by unecessarily defining thread_local when it isn't even used on + # Windows. + list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0) +else() + if(NOT HAS_THREAD_LOCAL) + list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread) + endif() endif() set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS}) +add_compiler_rt_component(fuzzer) + add_compiler_rt_object_libraries(RTfuzzer OS ${FUZZER_SUPPORTED_OS} ARCHS ${FUZZER_SUPPORTED_ARCH} @@ -108,12 +117,19 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main CFLAGS ${LIBFUZZER_CFLAGS} PARENT_TARGET fuzzer) -if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) +if(OS_NAME MATCHES "Linux|Fuchsia" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) macro(partially_link_libcxx name dir arch) + if(${arch} MATCHES "i386") + set(EMULATION_ARGUMENT "-m" "elf_i386") + else() + set(EMULATION_ARGUMENT "") + endif() set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir") file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir}) add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD - COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o + COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o @@ -126,13 +142,8 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch}) add_custom_libcxx(libcxx_fuzzer_${arch} ${LIBCXX_${arch}_PREFIX} CFLAGS ${TARGET_CFLAGS} - -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1 - -fvisibility=hidden CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON - -DLIBCXX_ENABLE_EXCEPTIONS=OFF - -DLIBCXX_ENABLE_SHARED=OFF - -DLIBCXX_ABI_NAMESPACE=Fuzzer - -DLIBCXX_CXX_ABI=none) + -DLIBCXX_ABI_NAMESPACE=Fuzzer) target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build) target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) diff --git a/lib/fuzzer/FuzzerBuiltins.h b/lib/fuzzer/FuzzerBuiltins.h index a80938d9a..5f1ccef8a 100644 --- a/lib/fuzzer/FuzzerBuiltins.h +++ b/lib/fuzzer/FuzzerBuiltins.h @@ -1,9 +1,8 @@ //===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Wrapper functions and marcos around builtin functions. diff --git a/lib/fuzzer/FuzzerBuiltinsMsvc.h b/lib/fuzzer/FuzzerBuiltinsMsvc.h index 67dd57ff9..82709cfe7 100644 --- a/lib/fuzzer/FuzzerBuiltinsMsvc.h +++ b/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -1,9 +1,8 @@ //===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Wrapper functions and marcos that use intrinsics instead of builtin functions @@ -25,7 +24,7 @@ // __builtin_return_address() cannot be compiled with MSVC. Use the equivalent // from <intrin.h> -#define GET_CALLER_PC() reinterpret_cast<uintptr_t>(_ReturnAddress()) +#define GET_CALLER_PC() _ReturnAddress() namespace fuzzer { diff --git a/lib/fuzzer/FuzzerCommand.h b/lib/fuzzer/FuzzerCommand.h index 9d258a228..87308864a 100644 --- a/lib/fuzzer/FuzzerCommand.h +++ b/lib/fuzzer/FuzzerCommand.h @@ -1,9 +1,8 @@ //===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerCommand represents a command to run in a subprocess. It allows callers diff --git a/lib/fuzzer/FuzzerCorpus.h b/lib/fuzzer/FuzzerCorpus.h index f844c07c7..6a95ef3a8 100644 --- a/lib/fuzzer/FuzzerCorpus.h +++ b/lib/fuzzer/FuzzerCorpus.h @@ -1,9 +1,8 @@ //===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::InputCorpus @@ -86,9 +85,10 @@ class InputCorpus { bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } - void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, - bool HasFocusFunction, const Vector<uint32_t> &FeatureSet, - const DataFlowTrace &DFT, const InputInfo *BaseII) { + InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + bool HasFocusFunction, + const Vector<uint32_t> &FeatureSet, + const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); @@ -114,6 +114,7 @@ class InputCorpus { UpdateCorpusDistribution(); PrintCorpus(); // ValidateFeatureSet(); + return &II; } // Debug-only @@ -170,7 +171,7 @@ class InputCorpus { InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; assert(!II.U.empty()); return II; - }; + } // Returns an index of random unit from the corpus to mutate. size_t ChooseUnitIdxToMutate(Random &Rand) { diff --git a/lib/fuzzer/FuzzerCrossOver.cpp b/lib/fuzzer/FuzzerCrossOver.cpp index 8b0fd7d52..83d9f8d47 100644 --- a/lib/fuzzer/FuzzerCrossOver.cpp +++ b/lib/fuzzer/FuzzerCrossOver.cpp @@ -1,9 +1,8 @@ //===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Cross over test inputs. diff --git a/lib/fuzzer/FuzzerDataFlowTrace.cpp b/lib/fuzzer/FuzzerDataFlowTrace.cpp index 764f3e49f..5ae7510e3 100644 --- a/lib/fuzzer/FuzzerDataFlowTrace.cpp +++ b/lib/fuzzer/FuzzerDataFlowTrace.cpp @@ -1,9 +1,8 @@ //===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace diff --git a/lib/fuzzer/FuzzerDataFlowTrace.h b/lib/fuzzer/FuzzerDataFlowTrace.h index ad4faeab7..9523a01e1 100644 --- a/lib/fuzzer/FuzzerDataFlowTrace.h +++ b/lib/fuzzer/FuzzerDataFlowTrace.h @@ -1,9 +1,8 @@ //===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace; reads and handles a data-flow trace. diff --git a/lib/fuzzer/FuzzerDefs.h b/lib/fuzzer/FuzzerDefs.h index c3dccbcd8..320b37d5f 100644 --- a/lib/fuzzer/FuzzerDefs.h +++ b/lib/fuzzer/FuzzerDefs.h @@ -1,9 +1,8 @@ //===- FuzzerDefs.h - Internal header for the Fuzzer ------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Basic definitions. @@ -120,31 +119,39 @@ # define ALWAYS_INLINE #endif // __clang__ -#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) - -#if defined(__has_feature) -# if __has_feature(address_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS -# elif __has_feature(memory_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY -# else -# define ATTRIBUTE_NO_SANITIZE_ALL -# endif +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS #else -# define ATTRIBUTE_NO_SANITIZE_ALL +#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) #endif #if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) #define ATTRIBUTE_INTERFACE __declspec(dllexport) // This is used for __sancov_lowest_stack which is needed for // -fsanitize-coverage=stack-depth. That feature is not yet available on // Windows, so make the symbol static to avoid linking errors. -#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ - __attribute__((tls_model("initial-exec"))) thread_local static +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) #else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) #define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) #define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif + +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS +# elif __has_feature(memory_sanitizer) +# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY +# else +# define ATTRIBUTE_NO_SANITIZE_ALL +# endif +#else +# define ATTRIBUTE_NO_SANITIZE_ALL #endif namespace fuzzer { diff --git a/lib/fuzzer/FuzzerDictionary.h b/lib/fuzzer/FuzzerDictionary.h index 0d9d91bcd..301c5d9af 100644 --- a/lib/fuzzer/FuzzerDictionary.h +++ b/lib/fuzzer/FuzzerDictionary.h @@ -1,9 +1,8 @@ //===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Dictionary diff --git a/lib/fuzzer/FuzzerDriver.cpp b/lib/fuzzer/FuzzerDriver.cpp index ff2a639ac..b9c892747 100644 --- a/lib/fuzzer/FuzzerDriver.cpp +++ b/lib/fuzzer/FuzzerDriver.cpp @@ -1,9 +1,8 @@ //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerDriver and flag parsing. @@ -11,12 +10,13 @@ #include "FuzzerCommand.h" #include "FuzzerCorpus.h" +#include "FuzzerFork.h" #include "FuzzerIO.h" #include "FuzzerInterface.h" #include "FuzzerInternal.h" +#include "FuzzerMerge.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <atomic> @@ -26,10 +26,16 @@ #include <mutex> #include <string> #include <thread> +#include <fstream> // This function should be present in the libFuzzer so that the client // binary can test for its existence. +#if LIBFUZZER_MSVC +extern "C" void __libfuzzer_is_present() {} +#pragma comment(linker, "/include:__libfuzzer_is_present") +#else extern "C" __attribute__((used)) void __libfuzzer_is_present() {} +#endif // LIBFUZZER_MSVC namespace fuzzer { @@ -316,10 +322,8 @@ int CleanseCrashInput(const Vector<std::string> &Args, assert(Cmd.hasArgument(InputFilePath)); Cmd.removeArgument(InputFilePath); - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - auto TmpFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".repro"); + auto LogFilePath = TempPath(".txt"); + auto TmpFilePath = TempPath(".repro"); Cmd.addArgument(TmpFilePath); Cmd.setOutputFile(LogFilePath); Cmd.combineOutAndErr(); @@ -379,8 +383,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, BaseCmd.addFlag("max_total_time", "600"); } - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); + auto LogFilePath = TempPath(".txt"); BaseCmd.setOutputFile(LogFilePath); BaseCmd.combineOutAndErr(); @@ -464,6 +467,34 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { return 0; } +void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, + const Vector<std::string> &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); + exit(0); + } + + Vector<SizedFile> OldCorpus, NewCorpus; + GetSizedFilesFromDir(Corpora[0], &OldCorpus); + for (size_t i = 1; i < Corpora.size(); i++) + GetSizedFilesFromDir(Corpora[i], &NewCorpus); + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt"); + Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + + exit(0); +} + int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, UnitVector& Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", @@ -573,6 +604,9 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.UnitTimeoutSec = Flags.timeout; Options.ErrorExitCode = Flags.error_exitcode; Options.TimeoutExitCode = Flags.timeout_exitcode; + Options.IgnoreTimeouts = Flags.ignore_timeouts; + Options.IgnoreOOMs = Flags.ignore_ooms; + Options.IgnoreCrashes = Flags.ignore_crashes; Options.MaxTotalTimeSec = Flags.max_total_time; Options.DoCrossOver = Flags.cross_over; Options.MutateDepth = Flags.mutate_depth; @@ -617,7 +651,6 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.PrintFinalStats = Flags.print_final_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; - Options.DumpCoverage = Flags.dump_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; if (Flags.exit_on_item) @@ -626,6 +659,9 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.FocusFunction = Flags.focus_function; if (Flags.data_flow_trace) Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.features_dir) + Options.FeaturesDir = Flags.features_dir; + Options.LazyCounters = Flags.lazy_counters; unsigned Seed = Flags.seed; // Initialize Seed. @@ -669,34 +705,6 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.cleanse_crash) return CleanseCrashInput(Args, Options); -#if 0 // deprecated, to be removed. - if (auto Name = Flags.run_equivalence_server) { - SMR.Destroy(Name); - if (!SMR.Create(Name)) { - Printf("ERROR: can't create shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE SERVER UP\n"); - while (true) { - SMR.WaitClient(); - size_t Size = SMR.ReadByteArraySize(); - SMR.WriteByteArray(nullptr, 0); - const Unit tmp(SMR.GetByteArray(), SMR.GetByteArray() + Size); - F->ExecuteCallback(tmp.data(), tmp.size()); - SMR.PostServer(); - } - return 0; - } - - if (auto Name = Flags.use_equivalence_server) { - if (!SMR.Open(Name)) { - Printf("ERROR: can't open shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE CLIENT UP\n"); - } -#endif - if (DoPlainRun) { Options.SaveArtifacts = false; int Runs = std::max(1, Flags.runs); @@ -719,13 +727,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - if (Flags.merge) { - F->CrashResistantMerge(Args, *Inputs, - Flags.load_coverage_summary, - Flags.save_coverage_summary, - Flags.merge_control_file); - exit(0); - } + if (Flags.fork) + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) + Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { const size_t kDefaultMaxMergeLen = 1 << 20; @@ -757,7 +763,28 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - F->Loop(*Inputs); + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file + Vector<std::string> ExtraSeedFiles; + if (Flags.seed_inputs) { + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + else + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); + exit(1); + } + // Parse SeedInputs. + size_t comma_pos = 0; + while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { + ExtraSeedFiles.push_back(SeedInputs.substr(comma_pos + 1)); + SeedInputs = SeedInputs.substr(0, comma_pos); + } + ExtraSeedFiles.push_back(SeedInputs); + } + + F->Loop(*Inputs, ExtraSeedFiles); if (Flags.verbosity) Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), diff --git a/lib/fuzzer/FuzzerExtFunctions.def b/lib/fuzzer/FuzzerExtFunctions.def index 8bfffdde5..288a59ce3 100644 --- a/lib/fuzzer/FuzzerExtFunctions.def +++ b/lib/fuzzer/FuzzerExtFunctions.def @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.def - External functions --------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This defines the external function pointers that @@ -29,7 +28,7 @@ EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t, EXT_FUNC(__lsan_enable, void, (), false); EXT_FUNC(__lsan_disable, void, (), false); EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); -EXT_FUNC(__sanitizer_acquire_crash_state, bool, (), true); +EXT_FUNC(__sanitizer_acquire_crash_state, int, (), true); EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, (void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)), @@ -44,8 +43,6 @@ EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, size_t module_path_len,void **pc_offset), false); EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true); EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false); -EXT_FUNC(__sanitizer_dump_coverage, void, (const uintptr_t *, uintptr_t), - false); EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false); EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false); EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false); diff --git a/lib/fuzzer/FuzzerExtFunctions.h b/lib/fuzzer/FuzzerExtFunctions.h index 2672a3854..c88aac4e6 100644 --- a/lib/fuzzer/FuzzerExtFunctions.h +++ b/lib/fuzzer/FuzzerExtFunctions.h @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.h - Interface to external functions ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Defines an interface to (possibly optional) functions. diff --git a/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp b/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp index 06bddd5de..dcd713459 100644 --- a/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsDlsym.cpp - Interface to external functions ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for operating systems that support dlsym(). We only use it on diff --git a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp index 6a6ef4932..ea5b87bd5 100644 --- a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsWeak.cpp - Interface to external functions -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for Linux. This relies on the linker's support for weak diff --git a/lib/fuzzer/FuzzerExtFunctionsWindows.cpp b/lib/fuzzer/FuzzerExtFunctionsWindows.cpp index b01871439..55efe8f80 100644 --- a/lib/fuzzer/FuzzerExtFunctionsWindows.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsWindows.cpp @@ -1,9 +1,8 @@ //=== FuzzerExtWindows.cpp - Interface to external functions --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation of FuzzerExtFunctions for Windows. Uses alternatename when @@ -50,7 +49,7 @@ extern "C" { Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ exit(1); \ } \ - EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG; + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG #include "FuzzerExtFunctions.def" diff --git a/lib/fuzzer/FuzzerExtraCounters.cpp b/lib/fuzzer/FuzzerExtraCounters.cpp index c99cd89be..3f38f4fb7 100644 --- a/lib/fuzzer/FuzzerExtraCounters.cpp +++ b/lib/fuzzer/FuzzerExtraCounters.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Extra coverage counters defined by user code. diff --git a/lib/fuzzer/FuzzerFlags.def b/lib/fuzzer/FuzzerFlags.def index 91281c979..b4ec5f298 100644 --- a/lib/fuzzer/FuzzerFlags.def +++ b/lib/fuzzer/FuzzerFlags.def @@ -1,9 +1,8 @@ //===- FuzzerFlags.def - Run-time flags -------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the @@ -21,6 +20,9 @@ FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " "size up to max_len.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-seperated list.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") @@ -35,11 +37,16 @@ FUZZER_FLAG_INT( "If one unit runs more than this number of seconds the process will abort.") FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " "this exit code will be used.") -FUZZER_FLAG_INT(timeout_exitcode, 77, "When libFuzzer reports a timeout " +FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " "this exit code will be used.") FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " + "in a subprocess") +FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") +FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") +FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") @@ -49,13 +56,6 @@ FUZZER_FLAG_STRING(merge_control_file, "If a merge process gets killed it tries to leave this file " "in a state suitable for resuming the merge. " "By default a temporary file will be used.") -FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:" - " save coverage summary to a given file." - " Used with -merge=1") -FUZZER_FLAG_STRING(load_coverage_summary, "Experimental:" - " load coverage summary from a given file." - " Treat this coverage as belonging to the first corpus. " - " Used with -merge=1") FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" " crash input. Use with -runs=N or -max_total_time=N to limit " "the number attempts." @@ -68,6 +68,10 @@ FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" " Use with -exact_artifact_path to specify the output." ) FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") +FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") @@ -107,9 +111,7 @@ FUZZER_FLAG_INT(print_corpus_stats, 0, "If 1, print statistics on corpus elements at exit.") FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" " at exit.") -FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." - " If 1, dump coverage information as a" - " .sancov file at exit.") +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") @@ -120,6 +122,9 @@ FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(lazy_counters, 0, "If 1, a performance optimization is" + "enabled for the 8bit inline counters. " + "Requires that libFuzzer successfully installs its SEGV handler") FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " "if 2, close stderr; if 3, close both. " "Be careful, this will also close e.g. stderr of asan.") @@ -148,8 +153,6 @@ FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " FUZZER_FLAG_STRING(focus_function, "Experimental. " "Fuzzing will focus on inputs that trigger calls to this function") -FUZZER_DEPRECATED_FLAG(run_equivalence_server) -FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") diff --git a/lib/fuzzer/FuzzerFork.cpp b/lib/fuzzer/FuzzerFork.cpp new file mode 100644 index 000000000..dd16ec1e2 --- /dev/null +++ b/lib/fuzzer/FuzzerFork.cpp @@ -0,0 +1,360 @@ +//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Spawn and orchestrate separate fuzzing processes. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include <atomic> +#include <chrono> +#include <fstream> +#include <memory> +#include <mutex> +#include <queue> +#include <sstream> +#include <thread> + +namespace fuzzer { + +struct Stats { + size_t number_of_executed_units = 0; + size_t peak_rss_mb = 0; + size_t average_exec_per_sec = 0; +}; + +static Stats ParseFinalStatsFromLog(const std::string &LogPath) { + std::ifstream In(LogPath); + std::string Line; + Stats Res; + struct { + const char *Name; + size_t *Var; + } NameVarPairs[] = { + {"stat::number_of_executed_units:", &Res.number_of_executed_units}, + {"stat::peak_rss_mb:", &Res.peak_rss_mb}, + {"stat::average_exec_per_sec:", &Res.average_exec_per_sec}, + {nullptr, nullptr}, + }; + while (std::getline(In, Line, '\n')) { + if (Line.find("stat::") != 0) continue; + std::istringstream ISS(Line); + std::string Name; + size_t Val; + ISS >> Name >> Val; + for (size_t i = 0; NameVarPairs[i].Name; i++) + if (Name == NameVarPairs[i].Name) + *NameVarPairs[i].Var = Val; + } + return Res; +} + +struct FuzzJob { + // Inputs. + Command Cmd; + std::string CorpusDir; + std::string FeaturesDir; + std::string LogPath; + std::string SeedListPath; + std::string CFPath; + + // Fuzzing Outputs. + int ExitCode; + + ~FuzzJob() { + RemoveFile(CFPath); + RemoveFile(LogPath); + RemoveFile(SeedListPath); + RmDirRecursive(CorpusDir); + RmDirRecursive(FeaturesDir); + } +}; + +struct GlobalEnv { + Vector<std::string> Args; + Vector<std::string> CorpusDirs; + std::string MainCorpusDir; + std::string TempDir; + Set<uint32_t> Features, Cov; + Vector<std::string> Files; + Random *Rand; + std::chrono::system_clock::time_point ProcessStartTime; + int Verbosity = 0; + + size_t NumTimeouts = 0; + size_t NumOOMs = 0; + size_t NumCrashes = 0; + + + size_t NumRuns = 0; + + size_t secondsSinceProcessStartUp() const { + return std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now() - ProcessStartTime) + .count(); + } + + FuzzJob *CreateNewJob(size_t JobId) { + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("print_final_stats", "1"); + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); + + auto Job = new FuzzJob; + std::string Seeds; + if (size_t CorpusSubsetSize = + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) + for (size_t i = 0; i < CorpusSubsetSize; i++) + Seeds += (Seeds.empty() ? "" : ",") + + Files[Rand->SkewTowardsLast(Files.size())]; + if (!Seeds.empty()) { + Job->SeedListPath = std::to_string(JobId) + ".seeds"; + WriteToFile(Seeds, Job->SeedListPath); + Cmd.addFlag("seed_inputs", "@" + Job->SeedListPath); + } + Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); + Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); + Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId)); + Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); + + + Cmd.addArgument(Job->CorpusDir); + Cmd.addFlag("features_dir", Job->FeaturesDir); + + for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) { + RmDirRecursive(D); + MkDir(D); + } + + Cmd.setOutputFile(Job->LogPath); + Cmd.combineOutAndErr(); + + Job->Cmd = Cmd; + + if (Verbosity >= 2) + Printf("Job %zd/%p Created: %s\n", JobId, Job, + Job->Cmd.toString().c_str()); + // Start from very short runs and gradually increase them. + return Job; + } + + void RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector<SizedFile> TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. + GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; + FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); + auto FeatureBytes = FileToVector(FeatureFile, 0, false); + assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); + Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); + for (auto Ft : NewFeatures) { + if (!Features.count(Ft)) { + MergeCandidates.push_back(F); + break; + } + } + } + if (MergeCandidates.empty()) return; + + Vector<std::string> FilesToAdd; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); + WriteToFile(U, NewPath); + Files.push_back(NewPath); + } + Features.insert(NewFeatures.begin(), NewFeatures.end()); + Cov.insert(NewCov.begin(), NewCov.end()); + for (auto Idx : NewCov) + if (auto *TE = TPC.PCTableEntryByIdx(Idx)) + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); + + if (!FilesToAdd.empty() || Job->ExitCode != 0) + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds\n", NumRuns, + Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, + NumOOMs, NumTimeouts, NumCrashes, secondsSinceProcessStartUp()); + } +}; + +struct JobQueue { + std::queue<FuzzJob *> Qu; + std::mutex Mu; + + void Push(FuzzJob *Job) { + std::lock_guard<std::mutex> Lock(Mu); + Qu.push(Job); + } + FuzzJob *Pop() { + std::lock_guard<std::mutex> Lock(Mu); + if (Qu.empty()) return nullptr; + auto Job = Qu.front(); + Qu.pop(); + return Job; + } +}; + +void WorkerThread(std::atomic<bool> *Stop, JobQueue *FuzzQ, JobQueue *MergeQ) { + while (!Stop->load()) { + auto Job = FuzzQ->Pop(); + // Printf("WorkerThread: job %p\n", Job); + if (!Job) { + SleepSeconds(1); + continue; + } + Job->ExitCode = ExecuteCommand(Job->Cmd); + MergeQ->Push(Job); + } +} + +// This is just a skeleton of an experimental -fork=1 feature. +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); + + GlobalEnv Env; + Env.Args = Args; + Env.CorpusDirs = CorpusDirs; + Env.Rand = &Rand; + Env.Verbosity = Options.Verbosity; + Env.ProcessStartTime = std::chrono::system_clock::now(); + + Vector<SizedFile> SeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &SeedFiles); + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath(".dir"); + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + MkDir(Env.TempDir); + + + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); + + int ExitCode = 0; + + JobQueue FuzzQ, MergeQ; + std::atomic<bool> Stop(false); + + size_t JobId = 1; + Vector<std::thread> Threads; + for (int t = 0; t < NumJobs; t++) { + Threads.push_back(std::thread(WorkerThread, &Stop, &FuzzQ, &MergeQ)); + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + while (true) { + std::unique_ptr<FuzzJob> Job(MergeQ.Pop()); + if (!Job) { + if (Stop) + break; + SleepSeconds(1); + continue; + } + ExitCode = Job->ExitCode; + if (ExitCode == Options.InterruptExitCode) { + Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); + Stop = true; + break; + } + Fuzzer::MaybeExitGracefully(); + + Env.RunOneMergeJob(Job.get()); + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) + Env.NumTimeouts++; + else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) + Env.NumOOMs++; + else if (ExitCode != 0) { + Env.NumCrashes++; + if (Options.IgnoreCrashes) { + std::ifstream In(Job->LogPath); + std::string Line; + while (std::getline(In, Line, '\n')) + if (Line.find("ERROR:") != Line.npos || + Line.find("runtime error:") != Line.npos) + Printf("%s\n", Line.c_str()); + } else { + // And exit if we don't ignore this crash. + Printf("INFO: log from the inner process:\n%s", + FileToString(Job->LogPath).c_str()); + Stop = true; + } + } + + // Stop if we are over the time budget. + // This is not precise, since other threads are still running + // and we will wait while joining them. + // We also don't stop instantly: other jobs need to finish. + if (Options.MaxTotalTimeSec > 0 && !Stop && + Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) { + Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n", + Env.secondsSinceProcessStartUp()); + Stop = true; + } + if (!Stop && Env.NumRuns >= Options.MaxNumberOfRuns) { + Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n", + Env.NumRuns); + Stop = true; + } + + if (!Stop) + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + Stop = true; + + for (auto &T : Threads) + T.join(); + + // The workers have terminated. Don't try to remove the directory before they + // terminate to avoid a race condition preventing cleanup on Windows. + RmDirRecursive(Env.TempDir); + + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); + exit(ExitCode); +} + +} // namespace fuzzer diff --git a/lib/fuzzer/FuzzerFork.h b/lib/fuzzer/FuzzerFork.h new file mode 100644 index 000000000..b29a43e13 --- /dev/null +++ b/lib/fuzzer/FuzzerFork.h @@ -0,0 +1,24 @@ +//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FORK_H +#define LLVM_FUZZER_FORK_H + +#include "FuzzerDefs.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +#include <string> + +namespace fuzzer { +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs); +} // namespace fuzzer + +#endif // LLVM_FUZZER_FORK_H diff --git a/lib/fuzzer/FuzzerIO.cpp b/lib/fuzzer/FuzzerIO.cpp index c4c31e824..7e5ba30a2 100644 --- a/lib/fuzzer/FuzzerIO.cpp +++ b/lib/fuzzer/FuzzerIO.cpp @@ -1,17 +1,17 @@ //===- FuzzerIO.cpp - IO utils. -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions. //===----------------------------------------------------------------------===// -#include "FuzzerIO.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" #include <algorithm> #include <cstdarg> #include <fstream> @@ -61,10 +61,19 @@ void CopyFileToErr(const std::string &Path) { } void WriteToFile(const Unit &U, const std::string &Path) { + WriteToFile(U.data(), U.size(), Path); +} + +void WriteToFile(const std::string &Data, const std::string &Path) { + WriteToFile(reinterpret_cast<const uint8_t *>(Data.c_str()), Data.size(), + Path); +} + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { // Use raw C interface because this function may be called from a sig handler. - FILE *Out = fopen(Path.c_str(), "w"); + FILE *Out = fopen(Path.c_str(), "wb"); if (!Out) return; - fwrite(U.data(), sizeof(U[0]), U.size(), Out); + fwrite(Data, sizeof(Data[0]), Size, Out); fclose(Out); } @@ -126,4 +135,25 @@ void Printf(const char *Fmt, ...) { fflush(OutputFile); } +void VPrintf(bool Verbose, const char *Fmt, ...) { + if (!Verbose) return; + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void RmDirRecursive(const std::string &Dir) { + IterateDirRecursive( + Dir, [](const std::string &Path) {}, + [](const std::string &Path) { RmDir(Path); }, + [](const std::string &Path) { RemoveFile(Path); }); +} + +std::string TempPath(const char *Extension) { + return DirPlusFile(TmpDir(), + "libFuzzerTemp." + std::to_string(GetPid()) + Extension); +} + } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerIO.h b/lib/fuzzer/FuzzerIO.h index b4a68190e..fe0d7b451 100644 --- a/lib/fuzzer/FuzzerIO.h +++ b/lib/fuzzer/FuzzerIO.h @@ -1,9 +1,8 @@ //===- FuzzerIO.h - Internal header for IO utils ----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO interface. @@ -25,6 +24,9 @@ std::string FileToString(const std::string &Path); void CopyFileToErr(const std::string &Path); +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); +// Write Data.c_str() to the file without terminating null character. +void WriteToFile(const std::string &Data, const std::string &Path); void WriteToFile(const Unit &U, const std::string &Path); void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, @@ -40,6 +42,8 @@ std::string DirName(const std::string &FileName); // Returns path to a TmpDir. std::string TmpDir(); +std::string TempPath(const char *Extension); + bool IsInterestingCoverageFile(const std::string &FileName); void DupAndCloseStderr(); @@ -47,6 +51,7 @@ void DupAndCloseStderr(); void CloseStdout(); void Printf(const char *Fmt, ...); +void VPrintf(bool Verbose, const char *Fmt, ...); // Print using raw syscalls, useful when printing at early init stages. void RawPrint(const char *Str); @@ -58,6 +63,16 @@ size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector<std::string> *V, bool TopDir); +void RmDirRecursive(const std::string &Dir); + +// Iterate files and dirs inside Dir, recursively. +// Call DirPreCallback/DirPostCallback on dirs before/after +// calling FileCallback on files. +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)); + struct SizedFile { std::string File; size_t Size; @@ -77,11 +92,17 @@ int CloseFile(int Fd); int DuplicateFile(int Fd); void RemoveFile(const std::string &Path); +void RenameFile(const std::string &OldPath, const std::string &NewPath); void DiscardOutput(int Fd); intptr_t GetHandleFromFd(int fd); +void MkDir(const std::string &Path); +void RmDir(const std::string &Path); + +const std::string &getDevNull(); + } // namespace fuzzer #endif // LLVM_FUZZER_IO_H diff --git a/lib/fuzzer/FuzzerIOPosix.cpp b/lib/fuzzer/FuzzerIOPosix.cpp index 401b4cbbf..cfd69bbc8 100644 --- a/lib/fuzzer/FuzzerIOPosix.cpp +++ b/lib/fuzzer/FuzzerIOPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOPosix.cpp - IO utils for Posix. ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation using Posix API. @@ -79,6 +78,28 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + DirPreCallback(Dir); + DIR *D = opendir(Dir.c_str()); + if (!D) return; + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + FileCallback(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } + closedir(D); + DirPostCallback(Dir); +} + char GetSeparator() { return '/'; } @@ -99,6 +120,10 @@ void RemoveFile(const std::string &Path) { unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("/dev/null", "w"); if (!Temp) @@ -137,11 +162,23 @@ bool IsInterestingCoverageFile(const std::string &FileName) { return true; } - void RawPrint(const char *Str) { write(2, Str, strlen(Str)); } +void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); +} + +void RmDir(const std::string &Path) { + rmdir(Path.c_str()); +} + +const std::string &getDevNull() { + static const std::string devNull = "/dev/null"; + return devNull; +} + } // namespace fuzzer #endif // LIBFUZZER_POSIX diff --git a/lib/fuzzer/FuzzerIOWindows.cpp b/lib/fuzzer/FuzzerIOWindows.cpp index 75dcaf72a..510afebef 100644 --- a/lib/fuzzer/FuzzerIOWindows.cpp +++ b/lib/fuzzer/FuzzerIOWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOWindows.cpp - IO utils for Windows. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation for Windows. @@ -72,6 +71,11 @@ bool IsFile(const std::string &Path) { return IsFile(Path, Att); } +static bool IsDir(DWORD FileAttrs) { + if (FileAttrs == INVALID_FILE_ATTRIBUTES) return false; + return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; +} + std::string Basename(const std::string &Path) { size_t Pos = Path.find_last_of("/\\"); if (Pos == std::string::npos) return Path; @@ -82,8 +86,10 @@ std::string Basename(const std::string &Path) { size_t FileSize(const std::string &Path) { WIN32_FILE_ATTRIBUTE_DATA attr; if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { - Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", - Path.c_str(), GetLastError()); + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), LastError); return 0; } ULARGE_INTEGER size; @@ -141,6 +147,58 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + // TODO(metzman): Implement ListFilesInDirRecursive via this function. + DirPreCallback(Dir); + + DWORD DirAttrs = GetFileAttributesA(Dir.c_str()); + if (!IsDir(DirAttrs)) return; + + std::string TargetDir(Dir); + assert(!TargetDir.empty()); + if (TargetDir.back() != '\\') TargetDir.push_back('\\'); + TargetDir.push_back('*'); + + WIN32_FIND_DATAA FindInfo; + // Find the directory's first file. + HANDLE FindHandle = FindFirstFileA(TargetDir.c_str(), &FindInfo); + if (FindHandle == INVALID_HANDLE_VALUE) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) { + // If the directory isn't empty, then something abnormal is going on. + Printf("FindFirstFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + } + return; + } + + do { + std::string Path = DirPlusFile(Dir, FindInfo.cFileName); + DWORD PathAttrs = FindInfo.dwFileAttributes; + if (IsDir(PathAttrs)) { + // Is Path the current directory (".") or the parent ("..")? + if (strcmp(FindInfo.cFileName, ".") == 0 || + strcmp(FindInfo.cFileName, "..") == 0) + continue; + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } else if (PathAttrs != INVALID_FILE_ATTRIBUTES) { + FileCallback(Path); + } + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + + FindClose(FindHandle); + DirPostCallback(Dir); +} + char GetSeparator() { return '\\'; } @@ -161,6 +219,10 @@ void RemoveFile(const std::string &Path) { _unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("nul", "w"); if (!Temp) @@ -334,8 +396,24 @@ bool IsInterestingCoverageFile(const std::string &FileName) { } void RawPrint(const char *Str) { - // Not tested, may or may not work. Fix if needed. - Printf("%s", Str); + _write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + if (CreateDirectoryA(Path.c_str(), nullptr)) return; + Printf("CreateDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +void RmDir(const std::string &Path) { + if (RemoveDirectoryA(Path.c_str())) return; + Printf("RemoveDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +const std::string &getDevNull() { + static const std::string devNull = "NUL"; + return devNull; } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerInterface.h b/lib/fuzzer/FuzzerInterface.h index 0f7effb2a..4f62822ea 100644 --- a/lib/fuzzer/FuzzerInterface.h +++ b/lib/fuzzer/FuzzerInterface.h @@ -1,9 +1,8 @@ //===- FuzzerInterface.h - Interface header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the interface between libFuzzer and the library being tested. @@ -26,25 +25,32 @@ extern "C" { #endif // __cplusplus +// Define FUZZER_INTERFACE_VISIBILITY to set default visibility in a way that +// doesn't break MSVC. +#if defined(_WIN32) +#define FUZZER_INTERFACE_VISIBILITY __declspec(dllexport) +#else +#define FUZZER_INTERFACE_VISIBILITY __attribute__((visibility("default"))) +#endif + // Mandatory user-provided target function. // Executes the code under test with [Data, Data+Size) as the input. // libFuzzer will invoke this function *many* times with different inputs. // Must return 0. -__attribute__((visibility("default"))) int +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); // Optional user-provided initialization function. // If provided, this function will be called by libFuzzer once at startup. // It may read and modify argc/argv. // Must return 0. -__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc, - char ***argv); +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerInitialize(int *argc, char ***argv); // Optional user-provided custom mutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. // Given the same Seed produces the same mutation. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed); @@ -52,7 +58,7 @@ LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, // Combines pieces of Data1 & Data2 together into Out. // Returns the new size, which is not greater than MaxOutSize. // Should produce the same mutation given the same Seed. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, size_t Size2, uint8_t *Out, size_t MaxOutSize, unsigned int Seed); @@ -61,9 +67,11 @@ LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); +#undef FUZZER_INTERFACE_VISIBILITY + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/lib/fuzzer/FuzzerInternal.h b/lib/fuzzer/FuzzerInternal.h index a7fdc89cb..f20dae014 100644 --- a/lib/fuzzer/FuzzerInternal.h +++ b/lib/fuzzer/FuzzerInternal.h @@ -1,9 +1,8 @@ //===- FuzzerInternal.h - Internal header for the Fuzzer --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the main class fuzzer::Fuzzer and most functions. @@ -36,8 +35,10 @@ public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(const Vector<std::string> &CorpusDirs); - void ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs); + void Loop(const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles); + void ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -72,11 +73,6 @@ public: // Merge Corpora[1:] into Corpora[0]. void Merge(const Vector<std::string> &Corpora); - void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<std::string> &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull); void CrashResistantMergeInternalStep(const std::string &ControlFilePath); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); @@ -90,20 +86,19 @@ public: bool DuringInitialCorpusExecution); void HandleMalloc(size_t Size); - void AnnounceOutput(const uint8_t *Data, size_t Size); + static void MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); private: void AlarmCallback(); void CrashCallback(); void ExitCallback(); - void MaybeExitGracefully(); void CrashOnOverwrittenData(); void InterruptCallback(); void MutateAndTestOne(); void PurgeAllocator(); void ReportNewCoverage(InputInfo *II, const Unit &U); void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); - void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); void PrintStatusForNewUnit(const Unit &U, const char *Text); diff --git a/lib/fuzzer/FuzzerLoop.cpp b/lib/fuzzer/FuzzerLoop.cpp index a32a30723..fd5b226a1 100644 --- a/lib/fuzzer/FuzzerLoop.cpp +++ b/lib/fuzzer/FuzzerLoop.cpp @@ -1,9 +1,8 @@ //===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Fuzzer's main loop. @@ -14,7 +13,6 @@ #include "FuzzerInternal.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <cstring> @@ -41,8 +39,6 @@ static const size_t kMaxUnitSizeToPrint = 256; thread_local bool Fuzzer::IsMyThread; -SharedMemoryRegion SMR; - bool RunningUserCallback = false; // Only one Fuzzer per process. @@ -135,7 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, @@ -157,7 +153,7 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); MaxInputLen = MaxMutationLen = Options.MaxLen; - TmpMaxMutationLen = Max(size_t(4), Corpus.MaxInputSize()); + TmpMaxMutationLen = 0; // Will be set once we load the corpus. AllocateCurrentUnitData(); CurrentUnitSize = 0; memset(BaseSha1, 0, sizeof(BaseSha1)); @@ -231,8 +227,9 @@ void Fuzzer::StaticFileSizeExceedCallback() { } void Fuzzer::CrashCallback() { - if (EF->__sanitizer_acquire_crash_state) - EF->__sanitizer_acquire_crash_state(); + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); PrintStackTrace(); Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" @@ -259,16 +256,20 @@ void Fuzzer::ExitCallback() { } void Fuzzer::MaybeExitGracefully() { - if (!GracefulExitRequested) return; + if (!F->GracefulExitRequested) return; Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); - PrintFinalStats(); + RmDirRecursive(TempPath(".dir")); + F->PrintFinalStats(); _Exit(0); } void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); - _Exit(0); // Stop right now, don't perform any at-exit actions. + ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). + RmDirRecursive(TempPath(".dir")); + // Stop right now, don't perform any at-exit actions. + _Exit(Options.InterruptExitCode); } NO_SANITIZE_MEMORY @@ -317,7 +318,7 @@ void Fuzzer::RssLimitCallback() { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { @@ -355,8 +356,6 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); - if (Options.DumpCoverage) - TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); if (!Options.PrintFinalStats) @@ -388,10 +387,10 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { static auto *PCsSet = new Set<uintptr_t>; - auto HandlePC = [&](uintptr_t PC) { - if (!PCsSet->insert(PC).second) + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) return; - std::string Descr = DescribePC("%F %L", PC + 1); + std::string Descr = DescribePC("%F %L", TE->PC + 1); if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { Printf("INFO: found line matching '%s', exiting.\n", Options.ExitOnSrcPos.c_str()); @@ -447,6 +446,23 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { } } +static void WriteFeatureSetToFile(const std::string &FeaturesDir, + const std::string &FileName, + const Vector<uint32_t> &FeatureSet) { + if (FeaturesDir.empty() || FeatureSet.empty()) return; + WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), + FeatureSet.size() * sizeof(FeatureSet[0]), + DirPlusFile(FeaturesDir, FileName)); +} + +static void RenameFeatureSetFile(const std::string &FeaturesDir, + const std::string &OldFile, + const std::string &NewFile) { + if (FeaturesDir.empty()) return; + RenameFile(DirPlusFile(FeaturesDir, OldFile), + DirPlusFile(FeaturesDir, NewFile)); +} + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, InputInfo *II, bool *FoundUniqFeatures) { if (!Size) @@ -471,15 +487,21 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; if (NumNewFeatures) { TPC.UpdateObservedPCs(); - Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, - TPC.ObservedFocusFunction(), UniqFeatureSetTmp, DFT, II); + auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, + MayDeleteFile, TPC.ObservedFocusFunction(), + UniqFeatureSetTmp, DFT, II); + WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), + NewII->UniqFeatureSet); return true; } if (II && FoundUniqFeaturesOfII && II->DataFlowTraceForFocusFunction.empty() && FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { + auto OldFeaturesFile = Sha1ToString(II->Sha1); Corpus.Replace(II, {Data, Data + Size}); + RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, + Sha1ToString(II->Sha1)); return true; } return false; @@ -513,8 +535,6 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); - if (SMR.IsClient()) - SMR.WriteByteArray(Data, Size); // We copy the contents of Unit into a separate heap buffer // so that we reliably find buffer overflows in it. uint8_t *DataCopy = new uint8_t[Size]; @@ -543,15 +563,16 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { delete[] DataCopy; } -void Fuzzer::WriteToOutputCorpus(const Unit &U) { +std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { if (Options.OnlyASCII) assert(IsASCII(U)); if (Options.OutputCorpus.empty()) - return; + return ""; std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); WriteToFile(U, Path); if (Options.Verbosity >= 2) Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); + return Path; } void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { @@ -637,6 +658,8 @@ void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); @@ -659,7 +682,9 @@ void Fuzzer::MutateAndTestOne() { Size <= CurrentMaxMutationLen) NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, II.DataFlowTraceForFocusFunction); - else + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); @@ -695,7 +720,9 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { +void Fuzzer::ReadAndExecuteSeedCorpora( + const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; Vector<SizedFile> SizedFiles; @@ -709,6 +736,11 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { Dir.c_str()); LastNumFiles = SizedFiles.size(); } + // Add files from -seed_inputs. + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + for (auto &File : SizedFiles) { MaxSize = Max(File.Size, MaxSize); MinSize = Min(File.Size, MinSize); @@ -722,6 +754,10 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { uint8_t dummy = 0; ExecuteCallback(&dummy, 0); + // Protect lazy counters here, after the once-init code has been executed. + if (Options.LazyCounters) + TPC.ProtectLazyCounters(); + if (SizedFiles.empty()) { Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); Unit U({'\n'}); // Valid ASCII input. @@ -764,14 +800,17 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { } } -void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) { - ReadAndExecuteSeedCorpora(CorpusDirs); +void Fuzzer::Loop(const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles) { + ReadAndExecuteSeedCorpora(CorpusDirs, ExtraSeedFiles); DFT.Clear(); // No need for DFT any more. TPC.SetPrintNewPCs(Options.PrintNewCovPcs); TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); system_clock::time_point LastCorpusReload = system_clock::now(); - if (Options.DoCrossOver) - MD.SetCorpus(&Corpus); + + TmpMaxMutationLen = + Min(MaxMutationLen, Max(size_t(4), Corpus.MaxInputSize())); + while (true) { auto Now = system_clock::now(); if (duration_cast<seconds>(Now - LastCorpusReload).count() >= @@ -824,44 +863,14 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { } } -void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) { - if (SMR.IsServer()) { - SMR.WriteByteArray(Data, Size); - } else if (SMR.IsClient()) { - SMR.PostClient(); - SMR.WaitServer(); - size_t OtherSize = SMR.ReadByteArraySize(); - uint8_t *OtherData = SMR.GetByteArray(); - if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) { - size_t i = 0; - for (i = 0; i < Min(Size, OtherSize); i++) - if (Data[i] != OtherData[i]) - break; - Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; " - "offset %zd\n", - GetPid(), Size, OtherSize, i); - DumpCurrentUnit("mismatch-"); - Printf("SUMMARY: libFuzzer: equivalence-mismatch\n"); - PrintFinalStats(); - _Exit(Options.ErrorExitCode); - } - } -} - } // namespace fuzzer extern "C" { -__attribute__((visibility("default"))) size_t +ATTRIBUTE_INTERFACE size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { assert(fuzzer::F); return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); } -// Experimental -__attribute__((visibility("default"))) void -LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) { - assert(fuzzer::F); - fuzzer::F->AnnounceOutput(Data, Size); -} } // extern "C" diff --git a/lib/fuzzer/FuzzerMain.cpp b/lib/fuzzer/FuzzerMain.cpp index f2c8e9c7b..771a34aed 100644 --- a/lib/fuzzer/FuzzerMain.cpp +++ b/lib/fuzzer/FuzzerMain.cpp @@ -1,9 +1,8 @@ //===- FuzzerMain.cpp - main() function and flags -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // main() and flags. @@ -16,6 +15,6 @@ extern "C" { int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); } // extern "C" -__attribute__((visibility("default"))) int main(int argc, char **argv) { +ATTRIBUTE_INTERFACE int main(int argc, char **argv) { return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); } diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 5f3052a39..dace45ece 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -1,9 +1,8 @@ //===- FuzzerMerge.cpp - merging corpora ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging corpora. @@ -43,10 +42,12 @@ void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { // file1 // file2 # One file name per line. // STARTED 0 123 # FileID, file size -// DONE 0 1 4 6 8 # FileID COV1 COV2 ... -// STARTED 1 456 # If DONE is missing, the input crashed while processing. +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. // STARTED 2 567 -// DONE 2 8 9 +// FT 2 8 9 +// COV 2 11 12 bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastFailure.clear(); std::string Line; @@ -71,11 +72,12 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { if (!std::getline(IS, Files[i].Name, '\n')) return false; - // Parse STARTED and DONE lines. + // Parse STARTED, FT, and COV lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector<uint32_t> TmpFeatures; + Set<uint32_t> PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -90,19 +92,25 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; - } else if (Marker == "DONE") { - // DONE FILE_ID COV1 COV2 COV3 ... + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; LastSeenStartMarker = kInvalidStartMarker; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. - while (ISS1 >> std::hex >> N) + while (ISS1 >> N) TmpFeatures.push_back(N); std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; } + } else if (Marker == "COV") { + size_t CurrentFileIdx = N; + if (ParseCoverage) + while (ISS1 >> N) + if (PCs.insert(N).second) + Files[CurrentFileIdx].Cov.push_back(N); } else { return false; } @@ -121,21 +129,21 @@ size_t Merger::ApproximateMemoryConsumption() const { return Res; } -// Decides which files need to be merged (add thost to NewFiles). +// Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, Vector<std::string> *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set<uint32_t> AllFeatures(InitialFeatures); + Set<uint32_t> AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; AllFeatures.insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures.size(); - // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; @@ -161,22 +169,20 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, auto &Cur = Files[i].Features; // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), // Files[i].Size, Cur.size()); - size_t OldSize = AllFeatures.size(); - AllFeatures.insert(Cur.begin(), Cur.end()); - if (AllFeatures.size() > OldSize) + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); } - return AllFeatures.size() - InitialNumFeatures; -} - -void Merger::PrintSummary(std::ostream &OS) { - for (auto &File : Files) { - OS << std::hex; - OS << File.Name << " size: " << File.Size << " features: "; - for (auto Feature : File.Features) - OS << " " << Feature; - OS << "\n"; - } + return NewFeatures->size(); } Set<uint32_t> Merger::AllFeatures() const { @@ -186,25 +192,6 @@ Set<uint32_t> Merger::AllFeatures() const { return S; } -Set<uint32_t> Merger::ParseSummary(std::istream &IS) { - std::string Line, Tmp; - Set<uint32_t> Res; - while (std::getline(IS, Line, '\n')) { - size_t N; - std::istringstream ISS1(Line); - ISS1 >> Tmp; // Name - ISS1 >> Tmp; // size: - assert(Tmp == "size:" && "Corrupt summary file"); - ISS1 >> std::hex; - ISS1 >> N; // File Size - ISS1 >> Tmp; // features: - assert(Tmp == "features:" && "Corrupt summary file"); - while (ISS1 >> std::hex >> N) - Res.insert(N); - } - return Res; -} - // Inner process. May crash if the target crashes. void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); @@ -223,8 +210,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set<size_t> AllFeatures; + Set<const TracePC::PCTableEntry *> AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { - MaybeExitGracefully(); + Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); if (U.size() > MaxInputLen) { U.resize(MaxInputLen); @@ -232,7 +220,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { } std::ostringstream StartedLine; // Write the pre-run marker. - OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; + OF << "STARTED " << i << " " << U.size() << "\n"; OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); @@ -247,26 +235,36 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { if (AllFeatures.insert(Feature).second) UniqFeatures.insert(Feature); }); + TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. - OF << "DONE " << i; + OF << "FT " << i; for (size_t F : UniqFeatures) - OF << " " << std::hex << F; + OF << " " << F; + OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); OF << "\n"; OF.flush(); } + PrintStats("DONE "); } static void WriteNewControlFile(const std::string &CFPath, - const Vector<SizedFile> &AllFiles, - size_t NumFilesInFirstCorpus) { + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus) { RemoveFile(CFPath); std::ofstream ControlFile(CFPath); - ControlFile << AllFiles.size() << "\n"; - ControlFile << NumFilesInFirstCorpus << "\n"; - for (auto &SF: AllFiles) + ControlFile << (OldCorpus.size() + NewCorpus.size()) << "\n"; + ControlFile << OldCorpus.size() << "\n"; + for (auto &SF: OldCorpus) + ControlFile << SF.File << "\n"; + for (auto &SF: NewCorpus) ControlFile << SF.File << "\n"; if (!ControlFile) { Printf("MERGE-OUTER: failed to write to the control file: %s\n", @@ -275,116 +273,89 @@ static void WriteNewControlFile(const std::string &CFPath, } } -// Outer process. Does not call the target code and thus sohuld not fail. -void Fuzzer::CrashResistantMerge(const Vector<std::string> &Args, - const Vector<std::string> &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull) { - if (Corpora.size() <= 1) { - Printf("Merge requires two or more corpus dirs\n"); - return; - } - auto CFPath = - MergeControlFilePathOrNull - ? MergeControlFilePathOrNull - : DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - +// Outer process. Does not call the target code and thus should not fail. +void CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) { - Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", - MergeControlFilePathOrNull); + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", + CFPath.c_str()); Merger M; - std::ifstream IF(MergeControlFilePathOrNull); + std::ifstream IF(CFPath); if (M.Parse(IF, /*ParseCoverage=*/false)) { - Printf("MERGE-OUTER: control file ok, %zd files total," + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," " first not processed file %zd\n", M.Files.size(), M.FirstNotProcessedFile); if (!M.LastFailure.empty()) - Printf("MERGE-OUTER: '%s' will be skipped as unlucky " + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " "(merge has stumbled on it the last time)\n", M.LastFailure.c_str()); if (M.FirstNotProcessedFile >= M.Files.size()) { - Printf("MERGE-OUTER: nothing to do, merge has been completed before\n"); + VPrintf( + V, "MERGE-OUTER: nothing to do, merge has been completed before\n"); exit(0); } NumAttempts = M.Files.size() - M.FirstNotProcessedFile; } else { - Printf("MERGE-OUTER: bad control file, will overwrite it\n"); + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); } } if (!NumAttempts) { // The supplied control file is empty or bad, create a fresh one. - Vector<SizedFile> AllFiles; - GetSizedFilesFromDir(Corpora[0], &AllFiles); - size_t NumFilesInFirstCorpus = AllFiles.size(); - std::sort(AllFiles.begin(), AllFiles.end()); - for (size_t i = 1; i < Corpora.size(); i++) - GetSizedFilesFromDir(Corpora[i], &AllFiles); - std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end()); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", - AllFiles.size(), NumFilesInFirstCorpus); - WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus); - NumAttempts = AllFiles.size(); + NumAttempts = OldCorpus.size() + NewCorpus.size(); + VPrintf(V, "MERGE-OUTER: %zd files, %zd in the initial corpus\n", + NumAttempts, OldCorpus.size()); + WriteNewControlFile(CFPath, OldCorpus, NewCorpus); } // Execute the inner process until it passes. // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); - bool Success = false; + BaseCmd.removeFlag("fork"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { - MaybeExitGracefully(); - Printf("MERGE-OUTER: attempt %zd\n", Attempt); + Fuzzer::MaybeExitGracefully(); + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); - Success = true; + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); break; } } - if (!Success) { - Printf("MERGE-OUTER: zero succesfull attempts, exiting\n"); - exit(1); - } // Read the control file and do the merge. Merger M; std::ifstream IF(CFPath); IF.seekg(0, IF.end); - Printf("MERGE-OUTER: the control file has %zd bytes\n", (size_t)IF.tellg()); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); IF.seekg(0, IF.beg); M.ParseOrExit(IF, true); IF.close(); - Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", - M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - if (CoverageSummaryOutputPathOrNull) { - Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n", - M.Files.size(), CoverageSummaryOutputPathOrNull); - std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); - M.PrintSummary(SummaryOut); - } - Vector<std::string> NewFiles; - Set<uint32_t> InitialFeatures; - if (CoverageSummaryInputPathOrNull) { - std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); - InitialFeatures = M.ParseSummary(SummaryIn); - Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", - CoverageSummaryInputPathOrNull, InitialFeatures.size()); - } - size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); - Printf("MERGE-OUTER: %zd new files with %zd new features added\n", - NewFiles.size(), NumNewFeatures); - for (auto &F: NewFiles) - WriteToOutputCorpus(FileToVector(F, MaxInputLen)); - // We are done, delete the control file if it was a temporary one. - if (!MergeControlFilePathOrNull) - RemoveFile(CFPath); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerMerge.h b/lib/fuzzer/FuzzerMerge.h index e54885a1e..c14dd589e 100644 --- a/lib/fuzzer/FuzzerMerge.h +++ b/lib/fuzzer/FuzzerMerge.h @@ -1,9 +1,8 @@ //===- FuzzerMerge.h - merging corpa ----------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging Corpora. @@ -52,7 +51,7 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector<uint32_t> Features; + Vector<uint32_t> Features, Cov; }; struct Merger { @@ -64,17 +63,24 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - void PrintSummary(std::ostream &OS); - Set<uint32_t> ParseSummary(std::istream &IS); - size_t Merge(const Set<uint32_t> &InitialFeatures, + size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, Vector<std::string> *NewFiles); - size_t Merge(Vector<std::string> *NewFiles) { - return Merge(Set<uint32_t>{}, NewFiles); - } size_t ApproximateMemoryConsumption() const; Set<uint32_t> AllFeatures() const; }; +void CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool Verbose); + } // namespace fuzzer #endif // LLVM_FUZZER_MERGE_H diff --git a/lib/fuzzer/FuzzerMutate.cpp b/lib/fuzzer/FuzzerMutate.cpp index 142b2b0b0..29541eac5 100644 --- a/lib/fuzzer/FuzzerMutate.cpp +++ b/lib/fuzzer/FuzzerMutate.cpp @@ -1,20 +1,19 @@ //===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Mutate a test input. //===----------------------------------------------------------------------===// -#include "FuzzerMutate.h" -#include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" +#include "FuzzerMutate.h" #include "FuzzerOptions.h" +#include "FuzzerTracePC.h" namespace fuzzer { @@ -73,10 +72,10 @@ size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { - if (!Corpus || Corpus->size() < 2 || Size == 0) + if (Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &Other = (*Corpus)[Idx]; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; if (Other.empty()) return 0; CustomCrossOverInPlaceHere.resize(MaxSize); @@ -422,9 +421,9 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { if (Size > MaxSize) return 0; - if (!Corpus || Corpus->size() < 2 || Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &O = (*Corpus)[Idx]; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; if (O.empty()) return 0; MutateInPlaceHere.resize(MaxSize); auto &U = MutateInPlaceHere; @@ -530,7 +529,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, const Vector<uint8_t> &Mask) { - assert(Size <= Mask.size()); + size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T // * Copy T back. @@ -539,16 +538,17 @@ size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, if (T.size() < Size) T.resize(Size); size_t OneBits = 0; - for (size_t I = 0; I < Size; I++) + for (size_t I = 0; I < MaskedSize; I++) if (Mask[I]) T[OneBits++] = Data[I]; + if (!OneBits) return 0; assert(!T.empty()); size_t NewSize = Mutate(T.data(), OneBits, OneBits); assert(NewSize <= OneBits); (void)NewSize; // Even if NewSize < OneBits we still use all OneBits bytes. - for (size_t I = 0, J = 0; I < Size; I++) + for (size_t I = 0, J = 0; I < MaskedSize; I++) if (Mask[I]) Data[I] = T[J++]; return Size; diff --git a/lib/fuzzer/FuzzerMutate.h b/lib/fuzzer/FuzzerMutate.h index a51c7fb44..6cbce8027 100644 --- a/lib/fuzzer/FuzzerMutate.h +++ b/lib/fuzzer/FuzzerMutate.h @@ -1,9 +1,8 @@ //===- FuzzerMutate.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::MutationDispatcher @@ -64,7 +63,7 @@ public: /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); - /// CrossOver Data with some other element of the corpus. + /// CrossOver Data with CrossOverWith. size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); /// Applies one of the configured mutations. @@ -89,7 +88,7 @@ public: void PrintRecommendedDictionary(); - void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } Random &GetRand() { return Rand; } @@ -140,7 +139,7 @@ public: DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; - const InputCorpus *Corpus = nullptr; + const Unit *CrossOverWith = nullptr; Vector<uint8_t> MutateInPlaceHere; Vector<uint8_t> MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call diff --git a/lib/fuzzer/FuzzerOptions.h b/lib/fuzzer/FuzzerOptions.h index ab90df82a..d48439daa 100644 --- a/lib/fuzzer/FuzzerOptions.h +++ b/lib/fuzzer/FuzzerOptions.h @@ -1,8 +1,7 @@ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::FuzzingOptions @@ -20,8 +19,13 @@ struct FuzzingOptions { size_t MaxLen = 0; size_t LenControl = 1000; int UnitTimeoutSec = 300; - int TimeoutExitCode = 77; + int TimeoutExitCode = 70; + int OOMExitCode = 71; + int InterruptExitCode = 72; int ErrorExitCode = 77; + bool IgnoreTimeouts = true; + bool IgnoreOOMs = true; + bool IgnoreCrashes = false; int MaxTotalTimeSec = 0; int RssLimitMb = 0; int MallocLimitMb = 0; @@ -47,6 +51,7 @@ struct FuzzingOptions { std::string ExitOnItem; std::string FocusFunction; std::string DataFlowTrace; + std::string FeaturesDir; bool SaveArtifacts = true; bool PrintNEW = true; // Print a status line when new units are found; bool PrintNewCovPcs = false; @@ -68,6 +73,7 @@ struct FuzzingOptions { bool HandleXfsz = false; bool HandleUsr1 = false; bool HandleUsr2 = false; + bool LazyCounters = false; }; } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerRandom.h b/lib/fuzzer/FuzzerRandom.h index 8a1aa3ef5..659283eee 100644 --- a/lib/fuzzer/FuzzerRandom.h +++ b/lib/fuzzer/FuzzerRandom.h @@ -1,9 +1,8 @@ //===- FuzzerRandom.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Random @@ -15,12 +14,17 @@ #include <random> namespace fuzzer { -class Random : public std::mt19937 { +class Random : public std::minstd_rand { public: - Random(unsigned int seed) : std::mt19937(seed) {} - result_type operator()() { return this->std::mt19937::operator()(); } + Random(unsigned int seed) : std::minstd_rand(seed) {} + result_type operator()() { return this->std::minstd_rand::operator()(); } size_t Rand() { return this->operator()(); } size_t RandBool() { return Rand() % 2; } + size_t SkewTowardsLast(size_t n) { + size_t T = this->operator()(n * n); + size_t Res = sqrt(T); + return Res; + } size_t operator()(size_t n) { return n ? Rand() % n : 0; } intptr_t operator()(intptr_t From, intptr_t To) { assert(From < To); diff --git a/lib/fuzzer/FuzzerSHA1.cpp b/lib/fuzzer/FuzzerSHA1.cpp index d2f8e811b..43e5e78cd 100644 --- a/lib/fuzzer/FuzzerSHA1.cpp +++ b/lib/fuzzer/FuzzerSHA1.cpp @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This code is taken from public domain @@ -32,7 +31,8 @@ namespace { // Added for LibFuzzer #ifdef __BIG_ENDIAN__ # define SHA_BIG_ENDIAN -#elif defined __LITTLE_ENDIAN__ +// Windows is always little endian and MSVC doesn't have <endian.h> +#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS /* override */ #elif defined __BYTE_ORDER # if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ diff --git a/lib/fuzzer/FuzzerSHA1.h b/lib/fuzzer/FuzzerSHA1.h index 3b5e6e807..05cbacda8 100644 --- a/lib/fuzzer/FuzzerSHA1.h +++ b/lib/fuzzer/FuzzerSHA1.h @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Internal header for the SHA1 utils --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // SHA1 utils. diff --git a/lib/fuzzer/FuzzerShmem.h b/lib/fuzzer/FuzzerShmem.h deleted file mode 100644 index 53568e0ac..000000000 --- a/lib/fuzzer/FuzzerShmem.h +++ /dev/null @@ -1,69 +0,0 @@ -//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// - -#ifndef LLVM_FUZZER_SHMEM_H -#define LLVM_FUZZER_SHMEM_H - -#include <algorithm> -#include <cstring> -#include <string> - -#include "FuzzerDefs.h" - -namespace fuzzer { - -class SharedMemoryRegion { - public: - bool Create(const char *Name); - bool Open(const char *Name); - bool Destroy(const char *Name); - uint8_t *GetData() { return Data; } - void PostServer() {Post(0);} - void WaitServer() {Wait(0);} - void PostClient() {Post(1);} - void WaitClient() {Wait(1);} - - size_t WriteByteArray(const uint8_t *Bytes, size_t N) { - assert(N <= kShmemSize - sizeof(N)); - memcpy(GetData(), &N, sizeof(N)); - memcpy(GetData() + sizeof(N), Bytes, N); - assert(N == ReadByteArraySize()); - return N; - } - size_t ReadByteArraySize() { - size_t Res; - memcpy(&Res, GetData(), sizeof(Res)); - return Res; - } - uint8_t *GetByteArray() { return GetData() + sizeof(size_t); } - - bool IsServer() const { return Data && IAmServer; } - bool IsClient() const { return Data && !IAmServer; } - -private: - - static const size_t kShmemSize = 1 << 22; - bool IAmServer; - std::string Path(const char *Name); - std::string SemName(const char *Name, int Idx); - void Post(int Idx); - void Wait(int Idx); - - bool Map(int fd); - uint8_t *Data = nullptr; - void *Semaphore[2]; -}; - -extern SharedMemoryRegion SMR; - -} // namespace fuzzer - -#endif // LLVM_FUZZER_SHMEM_H diff --git a/lib/fuzzer/FuzzerShmemFuchsia.cpp b/lib/fuzzer/FuzzerShmemFuchsia.cpp deleted file mode 100644 index e9ce50c2a..000000000 --- a/lib/fuzzer/FuzzerShmemFuchsia.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers -// are not currently supported. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" - -#if LIBFUZZER_FUCHSIA - -#include "FuzzerShmem.h" - -namespace fuzzer { - -bool SharedMemoryRegion::Create(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return false; -} - -void SharedMemoryRegion::Post(int Idx) {} - -void SharedMemoryRegion::Wait(int Idx) {} - -} // namespace fuzzer - -#endif // LIBFUZZER_FUCHSIA diff --git a/lib/fuzzer/FuzzerShmemPosix.cpp b/lib/fuzzer/FuzzerShmemPosix.cpp deleted file mode 100644 index 41a93f610..000000000 --- a/lib/fuzzer/FuzzerShmemPosix.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_POSIX - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include <errno.h> -#include <fcntl.h> -#include <semaphore.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - // When passing a name without a leading <slash> character to - // sem_open, the behaviour is unspecified in POSIX. Add a leading - // <slash> character for the name if there is no such one. - if (!Res.empty() && Res[0] != '/') - Res.insert(Res.begin(), '/'); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - Data = - (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); - if (Data == (uint8_t*)-1) - return false; - return true; -} - -bool SharedMemoryRegion::Create(const char *Name) { - int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777); - if (fd < 0) return false; - if (ftruncate(fd, kShmemSize) < 0) return false; - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - sem_unlink(SemName(Name, i).c_str()); - Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = true; - return true; -} - -bool SharedMemoryRegion::Open(const char *Name) { - int fd = open(Path(Name).c_str(), O_RDWR); - if (fd < 0) return false; - struct stat stat_res; - if (0 != fstat(fd, &stat_res)) - return false; - assert(stat_res.st_size == kShmemSize); - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = false; - return true; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return 0 == unlink(Path(Name).c_str()); -} - -void SharedMemoryRegion::Post(int Idx) { - assert(Idx == 0 || Idx == 1); - sem_post((sem_t*)Semaphore[Idx]); -} - -void SharedMemoryRegion::Wait(int Idx) { - assert(Idx == 0 || Idx == 1); - for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) { - // sem_wait may fail if interrupted by a signal. - sleep(i); - if (i) - Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i, - strerror(errno)); - if (i == 9) abort(); - } -} - -} // namespace fuzzer - -#endif // LIBFUZZER_POSIX diff --git a/lib/fuzzer/FuzzerShmemWindows.cpp b/lib/fuzzer/FuzzerShmemWindows.cpp deleted file mode 100644 index d330ebf4f..000000000 --- a/lib/fuzzer/FuzzerShmemWindows.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include <fcntl.h> -#include <stdio.h> -#include <sys/stat.h> -#include <sys/types.h> - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Create(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -void SharedMemoryRegion::Post(int Idx) { - assert(0 && "UNIMPLEMENTED"); -} - -void SharedMemoryRegion::Wait(int Idx) { - Semaphore[1] = nullptr; - assert(0 && "UNIMPLEMENTED"); -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/lib/fuzzer/FuzzerTracePC.cpp b/lib/fuzzer/FuzzerTracePC.cpp index 80b33105b..a2d3b7e7f 100644 --- a/lib/fuzzer/FuzzerTracePC.cpp +++ b/lib/fuzzer/FuzzerTracePC.cpp @@ -1,9 +1,8 @@ //===- FuzzerTracePC.cpp - PC tracing--------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Trace PCs. @@ -24,15 +23,6 @@ #include "FuzzerValueBitMap.h" #include <set> -// The coverage counters and PCs. -// These are declared as global variables named "__sancov_*" to simplify -// experiments with inlined instrumentation. -alignas(64) ATTRIBUTE_INTERFACE -uint8_t __sancov_trace_pc_guard_8bit_counters[fuzzer::TracePC::kNumPCs]; - -ATTRIBUTE_INTERFACE -uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; - // Used by -fsanitize-coverage=stack-depth to track stack depth ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; @@ -40,33 +30,80 @@ namespace fuzzer { TracePC TPC; -uint8_t *TracePC::Counters() const { - return __sancov_trace_pc_guard_8bit_counters; -} - -uintptr_t *TracePC::PCs() const { - return __sancov_trace_pc_pcs; -} - size_t TracePC::GetTotalPCCoverage() { - if (ObservedPCs.size()) - return ObservedPCs.size(); - size_t Res = 0; - for (size_t i = 1, N = GetNumPCs(); i < N; i++) - if (PCs()[i]) - Res++; - return Res; + return ObservedPCs.size(); } void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { if (Start == Stop) return; - if (NumModulesWithInline8bitCounters && - ModuleCounters[NumModulesWithInline8bitCounters-1].Start == Start) return; - assert(NumModulesWithInline8bitCounters < - sizeof(ModuleCounters) / sizeof(ModuleCounters[0])); - ModuleCounters[NumModulesWithInline8bitCounters++] = {Start, Stop}; - NumInline8bitCounters += Stop - Start; + if (NumModules && + Modules[NumModules - 1].Start() == Start) + return; + assert(NumModules < + sizeof(Modules) / sizeof(Modules[0])); + auto &M = Modules[NumModules++]; + uint8_t *AlignedStart = RoundUpByPage(Start); + uint8_t *AlignedStop = RoundDownByPage(Stop); + size_t NumFullPages = AlignedStop > AlignedStart ? + (AlignedStop - AlignedStart) / PageSize() : 0; + bool NeedFirst = Start < AlignedStart || !NumFullPages; + bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart; + M.NumRegions = NumFullPages + NeedFirst + NeedLast;; + assert(M.NumRegions > 0); + M.Regions = new Module::Region[M.NumRegions]; + assert(M.Regions); + size_t R = 0; + if (NeedFirst) + M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false}; + for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize()) + M.Regions[R++] = {P, P + PageSize(), true, true}; + if (NeedLast) + M.Regions[R++] = {AlignedStop, Stop, true, false}; + assert(R == M.NumRegions); + assert(M.Size() == (size_t)(Stop - Start)); + assert(M.Stop() == Stop); + assert(M.Start() == Start); + NumInline8bitCounters += M.Size(); +} + +// Mark all full page counter regions as PROT_NONE and set Enabled=false. +// The first time the instrumented code hits such a protected/disabled +// counter region we should catch a SEGV and call UnprotectLazyCounters, +// which will mark the page as PROT_READ|PROT_WRITE and set Enabled=true. +// +// Whenever other functions iterate over the counters they should ignore +// regions with Enabled=false. +void TracePC::ProtectLazyCounters() { + size_t NumPagesProtected = 0; + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage) return; + if (Mprotect(R.Start, R.Stop - R.Start, false)) { + R.Enabled = false; + NumPagesProtected++; + } + }); + if (NumPagesProtected) + Printf("INFO: %zd pages of counters where protected;" + " libFuzzer's SEGV handler must be installed\n", + NumPagesProtected); +} + +bool TracePC::UnprotectLazyCounters(void *CounterPtr) { + // Printf("UnprotectLazyCounters: %p\n", CounterPtr); + if (!CounterPtr) + return false; + bool Done = false; + uint8_t *Addr = reinterpret_cast<uint8_t *>(CounterPtr); + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage || R.Enabled || Done) return; + if (Addr >= R.Start && Addr < R.Stop) + if (Mprotect(R.Start, R.Stop - R.Start, true)) { + R.Enabled = true; + Done = true; + } + }); + return Done; } void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { @@ -78,38 +115,13 @@ void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { NumPCsInPCTables += E - B; } -void TracePC::HandleInit(uint32_t *Start, uint32_t *Stop) { - if (Start == Stop || *Start) return; - assert(NumModules < sizeof(Modules) / sizeof(Modules[0])); - for (uint32_t *P = Start; P < Stop; P++) { - NumGuards++; - if (NumGuards == kNumPCs) { - RawPrint( - "WARNING: The binary has too many instrumented PCs.\n" - " You may want to reduce the size of the binary\n" - " for more efficient fuzzing and precise coverage data\n"); - } - *P = NumGuards % kNumPCs; - } - Modules[NumModules].Start = Start; - Modules[NumModules].Stop = Stop; - NumModules++; -} - void TracePC::PrintModuleInfo() { - if (NumGuards) { - Printf("INFO: Loaded %zd modules (%zd guards): ", NumModules, NumGuards); - for (size_t i = 0; i < NumModules; i++) - Printf("%zd [%p, %p), ", Modules[i].Stop - Modules[i].Start, - Modules[i].Start, Modules[i].Stop); - Printf("\n"); - } - if (NumModulesWithInline8bitCounters) { + if (NumModules) { Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", - NumModulesWithInline8bitCounters, NumInline8bitCounters); - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) - Printf("%zd [%p, %p), ", ModuleCounters[i].Stop - ModuleCounters[i].Start, - ModuleCounters[i].Start, ModuleCounters[i].Stop); + NumModules, NumInline8bitCounters); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(), + Modules[i].Stop()); Printf("\n"); } if (NumPCTables) { @@ -121,8 +133,7 @@ void TracePC::PrintModuleInfo() { } Printf("\n"); - if ((NumGuards && NumGuards != NumPCsInPCTables) || - (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables)) { + if (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables) { Printf("ERROR: The size of coverage PC tables does not match the\n" "number of instrumented PCs. This might be a compiler bug,\n" "please contact the libFuzzer developers.\n" @@ -163,7 +174,7 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { /// \return the address of the next instruction. /// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cc` -inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { +ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { #if defined(__mips__) return PC + 8; #elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ @@ -176,41 +187,34 @@ inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { void TracePC::UpdateObservedPCs() { Vector<uintptr_t> CoveredFuncs; - auto ObservePC = [&](uintptr_t PC) { - if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", GetNextInstructionPc(PC)); + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); Printf("\n"); } }; - auto Observe = [&](const PCTableEntry &TE) { - if (TE.PCFlags & 1) - if (++ObservedFuncs[TE.PC] == 1 && NumPrintNewFuncs) - CoveredFuncs.push_back(TE.PC); - ObservePC(TE.PC); + auto Observe = [&](const PCTableEntry *TE) { + if (PcIsFuncEntry(TE)) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); }; if (NumPCsInPCTables) { if (NumInline8bitCounters == NumPCsInPCTables) { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - assert(Size == - (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++) - if (Beg[j]) - Observe(ModulePCTable[i].Start[j]); - } - } else if (NumGuards == NumPCsInPCTables) { - size_t GuardIdx = 1; for (size_t i = 0; i < NumModules; i++) { - uint32_t *Beg = Modules[i].Start; - size_t Size = Modules[i].Stop - Beg; - assert(Size == + auto &M = Modules[i]; + assert(M.Size() == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, GuardIdx++) - if (Counters()[GuardIdx]) - Observe(ModulePCTable[i].Start[j]); + for (size_t r = 0; r < M.NumRegions; r++) { + auto &R = M.Regions[r]; + if (!R.Enabled) continue; + for (uint8_t *P = R.Start; P < R.Stop; P++) + if (*P) + Observe(&ModulePCTable[i].Start[M.Idx(P)]); + } } } } @@ -223,6 +227,27 @@ void TracePC::UpdateObservedPCs() { } } +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; +} + +const TracePC::PCTableEntry *TracePC::PCTableEntryByIdx(uintptr_t Idx) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + size_t Size = M.Stop - M.Start; + if (Idx < Size) return &M.Start[Idx]; + Idx -= Size; + } + return nullptr; +} static std::string GetModuleName(uintptr_t PC) { char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? @@ -242,47 +267,38 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { auto ModuleName = GetModuleName(M.Start->PC); for (auto NextFE = M.Start; NextFE < M.Stop; ) { auto FE = NextFE; - assert((FE->PCFlags & 1) && "Not a function entry point"); + assert(PcIsFuncEntry(FE) && "Not a function entry point"); do { NextFE++; - } while (NextFE < M.Stop && !(NextFE->PCFlags & 1)); - if (ObservedFuncs.count(FE->PC)) - CB(FE, NextFE, ObservedFuncs[FE->PC]); + } while (NextFE < M.Stop && !(PcIsFuncEntry(NextFE))); + CB(FE, NextFE, ObservedFuncs[FE->PC]); } } } void TracePC::SetFocusFunction(const std::string &FuncName) { // This function should be called once. - assert(FocusFunction.first > NumModulesWithInline8bitCounters); + assert(!FocusFunctionCounterPtr); if (FuncName.empty()) return; - for (size_t M = 0; M < NumModulesWithInline8bitCounters; M++) { + for (size_t M = 0; M < NumModules; M++) { auto &PCTE = ModulePCTable[M]; size_t N = PCTE.Stop - PCTE.Start; for (size_t I = 0; I < N; I++) { - if (!(PCTE.Start[I].PCFlags & 1)) continue; // not a function entry. + if (!(PcIsFuncEntry(&PCTE.Start[I]))) continue; // not a function entry. auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC)); if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ') Name = Name.substr(3, std::string::npos); if (FuncName != Name) continue; Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); - FocusFunction = {M, I}; + FocusFunctionCounterPtr = Modules[M].Start() + I; return; } } } bool TracePC::ObservedFocusFunction() { - size_t I = FocusFunction.first; - size_t J = FocusFunction.second; - if (I >= NumModulesWithInline8bitCounters) - return false; - auto &MC = ModuleCounters[I]; - size_t Size = MC.Stop - MC.Start; - if (J >= Size) - return false; - return MC.Start[J] != 0; + return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; } void TracePC::PrintCoverage() { @@ -306,32 +322,24 @@ void TracePC::PrintCoverage() { if (FunctionStr.find("in ") == 0) FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); - size_t Line = std::stoul(LineStr); size_t NumEdges = Last - First; Vector<uintptr_t> UncoveredPCs; for (auto TE = First; TE < Last; TE++) - if (!ObservedPCs.count(TE->PC)) + if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); - Printf("COVERED_FUNC: hits: %zd", Counter); + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); - Printf(" %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(), Line); - for (auto PC: UncoveredPCs) - Printf(" UNCOVERED_PC: %s\n", - DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); }; IterateCoveredFunctions(CoveredFunctionCallback); } -void TracePC::DumpCoverage() { - if (EF->__sanitizer_dump_coverage) { - Vector<uintptr_t> PCsCopy(GetNumPCs()); - for (size_t i = 0; i < GetNumPCs(); i++) - PCsCopy[i] = PCs()[i] ? GetPreviousInstructionPc(PCs()[i]) : 0; - EF->__sanitizer_dump_coverage(PCsCopy.data(), PCsCopy.size()); - } -} - // Value profile. // We keep track of various values that affect control flow. // These values are inserted into a bit-set-based hash map. @@ -400,11 +408,10 @@ static size_t InternalStrnlen2(const char *S1, const char *S2) { } void TracePC::ClearInlineCounters() { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - memset(Beg, 0, Size); - } + IterateCounterRegions([](const Module::Region &R){ + if (R.Enabled) + memset(R.Start, 0, R.Stop - R.Start); + }); } ATTRIBUTE_NO_SANITIZE_ALL @@ -417,16 +424,25 @@ uintptr_t TracePC::GetMaxStackOffset() const { return InitialStack - __sancov_lowest_stack; // Stack grows down } +void WarnAboutDeprecatedInstrumentation(const char *flag) { + // Use RawPrint because Printf cannot be used on Windows before OutputFile is + // initialized. + RawPrint(flag); + RawPrint( + " is no longer supported by libFuzzer.\n" + "Please either migrate to a compiler that supports -fsanitize=fuzzer\n" + "or use an older version of libFuzzer\n"); + exit(1); +} + } // namespace fuzzer extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { - uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); - uint32_t Idx = *Guard; - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } // Best-effort support for -fsanitize-coverage=trace-pc, which is available @@ -434,15 +450,13 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc() { - uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); - uintptr_t Idx = PC & (((uintptr_t)1 << fuzzer::TracePC::kTracePcBits) - 1); - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation("-fsanitize-coverage=trace-pc"); } ATTRIBUTE_INTERFACE void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { - fuzzer::TPC.HandleInit(Start, Stop); + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } ATTRIBUTE_INTERFACE @@ -537,24 +551,44 @@ void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { uint64_t N = Cases[0]; uint64_t ValSizeInBits = Cases[1]; uint64_t *Vals = Cases + 2; - // Skip the most common and the most boring case. - if (Vals[N - 1] < 256 && Val < 256) + // Skip the most common and the most boring case: all switch values are small. + // We may want to skip this at compile-time, but it will make the + // instrumentation less general. + if (Vals[N - 1] < 256) + return; + // Also skip small inputs values, they won't give good signal. + if (Val < 256) return; uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); size_t i; - uint64_t Token = 0; + uint64_t Smaller = 0; + uint64_t Larger = ~(uint64_t)0; + // Find two switch values such that Smaller < Val < Larger. + // Use 0 and 0xfff..f as the defaults. for (i = 0; i < N; i++) { - Token = Val ^ Vals[i]; - if (Val < Vals[i]) + if (Val < Vals[i]) { + Larger = Vals[i]; break; + } + if (Val > Vals[i]) Smaller = Vals[i]; } - if (ValSizeInBits == 16) - fuzzer::TPC.HandleCmp(PC + i, static_cast<uint16_t>(Token), (uint16_t)(0)); - else if (ValSizeInBits == 32) - fuzzer::TPC.HandleCmp(PC + i, static_cast<uint32_t>(Token), (uint32_t)(0)); - else - fuzzer::TPC.HandleCmp(PC + i, Token, (uint64_t)(0)); + // Apply HandleCmp to {Val,Smaller} and {Val, Larger}, + // use i as the PC modifier for HandleCmp. + if (ValSizeInBits == 16) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint16_t>(Val), + (uint16_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint16_t>(Val), + (uint16_t)(Larger)); + } else if (ValSizeInBits == 32) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint32_t>(Val), + (uint32_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint32_t>(Val), + (uint32_t)(Larger)); + } else { + fuzzer::TPC.HandleCmp(PC + 2*i, Val, Smaller); + fuzzer::TPC.HandleCmp(PC + 2*i + 1, Val, Larger); + } } ATTRIBUTE_INTERFACE diff --git a/lib/fuzzer/FuzzerTracePC.h b/lib/fuzzer/FuzzerTracePC.h index 46d6c2488..4f5ebeb04 100644 --- a/lib/fuzzer/FuzzerTracePC.h +++ b/lib/fuzzer/FuzzerTracePC.h @@ -1,9 +1,8 @@ //===- FuzzerTracePC.h - Internal header for the Fuzzer ---------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::TracePC @@ -70,11 +69,6 @@ struct MemMemTable { class TracePC { public: - static const size_t kNumPCs = 1 << 21; - // How many bits of PC are used from __sanitizer_cov_trace_pc. - static const size_t kTracePcBits = 18; - - void HandleInit(uint32_t *Start, uint32_t *Stop); void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); @@ -89,8 +83,6 @@ class TracePC { void ResetMaps() { ValueProfileMap.Reset(); - if (NumModules) - memset(Counters(), 0, GetNumPCs()); ClearExtraCounters(); ClearInlineCounters(); } @@ -103,7 +95,6 @@ class TracePC { void PrintModuleInfo(); void PrintCoverage(); - void DumpCoverage(); template<class CallBack> void IterateCoveredFunctions(CallBack CB); @@ -116,14 +107,6 @@ class TracePC { TableOfRecentCompares<Word, 32> TORCW; MemMemTable<1024> MMT; - size_t GetNumPCs() const { - return NumGuards == 0 ? (1 << kTracePcBits) : Min(kNumPCs, NumGuards + 1); - } - uintptr_t GetPC(size_t Idx) { - assert(Idx < GetNumPCs()); - return PCs()[Idx]; - } - void RecordInitialStack(); uintptr_t GetMaxStackOffset() const; @@ -136,39 +119,63 @@ class TracePC { void SetFocusFunction(const std::string &FuncName); bool ObservedFocusFunction(); + void ProtectLazyCounters(); + bool UnprotectLazyCounters(void *CounterPtr); + + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + const PCTableEntry *PCTableEntryByIdx(uintptr_t Idx); + static uintptr_t GetNextInstructionPc(uintptr_t PC); + bool PcIsFuncEntry(const PCTableEntry *TE) { return TE->PCFlags & 1; } + private: bool UseCounters = false; uint32_t UseValueProfileMask = false; bool DoPrintNewPCs = false; size_t NumPrintNewFuncs = 0; + // Module represents the array of 8-bit counters split into regions + // such that every region, except maybe the first and the last one, is one + // full page. struct Module { - uint32_t *Start, *Stop; + struct Region { + uint8_t *Start, *Stop; + bool Enabled; + bool OneFullPage; + }; + Region *Regions; + size_t NumRegions; + uint8_t *Start() { return Regions[0].Start; } + uint8_t *Stop() { return Regions[NumRegions - 1].Stop; } + size_t Size() { return Stop() - Start(); } + size_t Idx(uint8_t *P) { + assert(P >= Start() && P < Stop()); + return P - Start(); + } }; Module Modules[4096]; size_t NumModules; // linker-initialized. - size_t NumGuards; // linker-initialized. - - struct { uint8_t *Start, *Stop; } ModuleCounters[4096]; - size_t NumModulesWithInline8bitCounters; // linker-initialized. size_t NumInline8bitCounters; - struct PCTableEntry { - uintptr_t PC, PCFlags; - }; + template <class Callback> + void IterateCounterRegions(Callback CB) { + for (size_t m = 0; m < NumModules; m++) + for (size_t r = 0; r < Modules[m].NumRegions; r++) + CB(Modules[m].Regions[r]); + } struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; size_t NumPCTables; size_t NumPCsInPCTables; - uint8_t *Counters() const; - uintptr_t *PCs() const; - - Set<uintptr_t> ObservedPCs; + Set<const PCTableEntry*> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. - std::pair<size_t, size_t> FocusFunction = {-1, -1}; // Module and PC IDs. + uint8_t *FocusFunctionCounterPtr = nullptr; ValueBitMap ValueProfileMap; uintptr_t InitialStack; @@ -177,7 +184,7 @@ private: template <class Callback> // void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); ATTRIBUTE_NO_SANITIZE_ALL -void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, +size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, size_t FirstFeature, Callback Handle8bitCounter) { typedef uintptr_t LargeType; const size_t Step = sizeof(LargeType) / sizeof(uint8_t); @@ -199,6 +206,7 @@ void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, for (; P < End; P++) if (uint8_t V = *P) Handle8bitCounter(FirstFeature, P - Begin, V); + return End - Begin; } // Given a non-zero Counter returns a number in the range [0,7]. @@ -229,10 +237,8 @@ unsigned CounterToFeature(T Counter) { template <class Callback> // void Callback(size_t Feature) ATTRIBUTE_NO_SANITIZE_ADDRESS -__attribute__((noinline)) +ATTRIBUTE_NOINLINE void TracePC::CollectFeatures(Callback HandleFeature) const { - uint8_t *Counters = this->Counters(); - size_t N = GetNumPCs(); auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) @@ -243,22 +249,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { size_t FirstFeature = 0; - if (!NumInline8bitCounters) { - ForEachNonZeroByte(Counters, Counters + N, FirstFeature, Handle8bitCounter); - FirstFeature += N * 8; - } - - if (NumInline8bitCounters) { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - ForEachNonZeroByte(ModuleCounters[i].Start, ModuleCounters[i].Stop, - FirstFeature, Handle8bitCounter); - FirstFeature += 8 * (ModuleCounters[i].Stop - ModuleCounters[i].Start); + for (size_t i = 0; i < NumModules; i++) { + for (size_t r = 0; r < Modules[i].NumRegions; r++) { + if (!Modules[i].Regions[r].Enabled) continue; + FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start, + Modules[i].Regions[r].Stop, + FirstFeature, Handle8bitCounter); } } - ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature, - Handle8bitCounter); - FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8; + FirstFeature += + 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), + FirstFeature, Handle8bitCounter); if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { diff --git a/lib/fuzzer/FuzzerUtil.cpp b/lib/fuzzer/FuzzerUtil.cpp index 6286f9a71..7aa84a1fa 100644 --- a/lib/fuzzer/FuzzerUtil.cpp +++ b/lib/fuzzer/FuzzerUtil.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtil.cpp - Misc utils ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils. diff --git a/lib/fuzzer/FuzzerUtil.h b/lib/fuzzer/FuzzerUtil.h index d2f1d5de4..0a127911d 100644 --- a/lib/fuzzer/FuzzerUtil.h +++ b/lib/fuzzer/FuzzerUtil.h @@ -1,9 +1,8 @@ //===- FuzzerUtil.h - Internal header for the Fuzzer Utils ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Util functions. @@ -53,6 +52,8 @@ void SetSignalHandler(const FuzzingOptions& Options); void SleepSeconds(int Seconds); +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite); + unsigned long GetPid(); size_t GetPeakRSSMb(); @@ -88,6 +89,20 @@ size_t SimpleFastHash(const uint8_t *Data, size_t Size); inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } +inline size_t PageSize() { return 4096; } +inline uint8_t *RoundUpByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = (X + Mask) & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} +inline uint8_t *RoundDownByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = X & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} + } // namespace fuzzer #endif // LLVM_FUZZER_UTIL_H diff --git a/lib/fuzzer/FuzzerUtilDarwin.cpp b/lib/fuzzer/FuzzerUtilDarwin.cpp index 4bfbc11a5..171db2357 100644 --- a/lib/fuzzer/FuzzerUtilDarwin.cpp +++ b/lib/fuzzer/FuzzerUtilDarwin.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Darwin. diff --git a/lib/fuzzer/FuzzerUtilFuchsia.cpp b/lib/fuzzer/FuzzerUtilFuchsia.cpp index cd48fefef..7b5c8f647 100644 --- a/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Fuchsia/Zircon APIs. @@ -288,6 +287,10 @@ void CrashHandler(zx_handle_t *Event) { } // namespace +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + // Platform specific functions. void SetSignalHandler(const FuzzingOptions &Options) { // Set up alarm handler if needed. diff --git a/lib/fuzzer/FuzzerUtilLinux.cpp b/lib/fuzzer/FuzzerUtilLinux.cpp index c103fd230..d5a15d19f 100644 --- a/lib/fuzzer/FuzzerUtilLinux.cpp +++ b/lib/fuzzer/FuzzerUtilLinux.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilLinux.cpp - Misc utils for Linux. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Linux. @@ -14,12 +13,18 @@ #include "FuzzerCommand.h" #include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> + namespace fuzzer { int ExecuteCommand(const Command &Cmd) { std::string CmdLine = Cmd.toString(); - return system(CmdLine.c_str()); + int exit_code = system(CmdLine.c_str()); + if (WIFEXITED(exit_code)) + return WEXITSTATUS(exit_code); + return exit_code; } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerUtilPosix.cpp b/lib/fuzzer/FuzzerUtilPosix.cpp index bc64d3293..110785d87 100644 --- a/lib/fuzzer/FuzzerUtilPosix.cpp +++ b/lib/fuzzer/FuzzerUtilPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilPosix.cpp - Misc utils for Posix. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Posix API. @@ -12,6 +11,7 @@ #if LIBFUZZER_POSIX #include "FuzzerIO.h" #include "FuzzerInternal.h" +#include "FuzzerTracePC.h" #include <cassert> #include <chrono> #include <cstring> @@ -19,6 +19,7 @@ #include <iomanip> #include <signal.h> #include <stdio.h> +#include <sys/mman.h> #include <sys/resource.h> #include <sys/syscall.h> #include <sys/time.h> @@ -32,6 +33,16 @@ static void AlarmHandler(int, siginfo_t *, void *) { Fuzzer::StaticAlarmCallback(); } +static void (*upstream_segv_handler)(int, siginfo_t *, void *); + +static void SegvHandler(int sig, siginfo_t *si, void *ucontext) { + assert(si->si_signo == SIGSEGV); + if (TPC.UnprotectLazyCounters(si->si_addr)) return; + if (upstream_segv_handler) + return upstream_segv_handler(sig, si, ucontext); + Fuzzer::StaticCrashSignalCallback(); +} + static void CrashHandler(int, siginfo_t *, void *) { Fuzzer::StaticCrashSignalCallback(); } @@ -56,8 +67,11 @@ static void SetSigaction(int signum, exit(1); } if (sigact.sa_flags & SA_SIGINFO) { - if (sigact.sa_sigaction) - return; + if (sigact.sa_sigaction) { + if (signum != SIGSEGV) + return; + upstream_segv_handler = sigact.sa_sigaction; + } } else { if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && sigact.sa_handler != SIG_ERR) @@ -65,6 +79,7 @@ static void SetSigaction(int signum, } sigact = {}; + sigact.sa_flags = SA_SIGINFO; sigact.sa_sigaction = callback; if (sigaction(signum, &sigact, 0)) { Printf("libFuzzer: sigaction failed with %d\n", errno); @@ -83,6 +98,11 @@ void SetTimer(int Seconds) { SetSigaction(SIGALRM, AlarmHandler); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return 0 == mprotect(Ptr, Size, + AllowReadWrite ? (PROT_READ | PROT_WRITE) : PROT_NONE); +} + void SetSignalHandler(const FuzzingOptions& Options) { if (Options.UnitTimeoutSec > 0) SetTimer(Options.UnitTimeoutSec / 2 + 1); @@ -91,7 +111,7 @@ void SetSignalHandler(const FuzzingOptions& Options) { if (Options.HandleTerm) SetSigaction(SIGTERM, InterruptHandler); if (Options.HandleSegv) - SetSigaction(SIGSEGV, CrashHandler); + SetSigaction(SIGSEGV, SegvHandler); if (Options.HandleBus) SetSigaction(SIGBUS, CrashHandler); if (Options.HandleAbrt) diff --git a/lib/fuzzer/FuzzerUtilWindows.cpp b/lib/fuzzer/FuzzerUtilWindows.cpp index 393b4768b..074e1eb42 100644 --- a/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/lib/fuzzer/FuzzerUtilWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilWindows.cpp - Misc utils for Windows. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation for Windows. @@ -86,11 +85,11 @@ void CALLBACK AlarmHandler(PVOID, BOOLEAN) { class TimerQ { HANDLE TimerQueue; public: - TimerQ() : TimerQueue(NULL) {}; + TimerQ() : TimerQueue(NULL) {} ~TimerQ() { if (TimerQueue) DeleteTimerQueueEx(TimerQueue, NULL); - }; + } void SetTimer(int Seconds) { if (!TimerQueue) { TimerQueue = CreateTimerQueue(); @@ -105,13 +104,17 @@ class TimerQ { Printf("libFuzzer: CreateTimerQueueTimer failed.\n"); exit(1); } - }; + } }; static TimerQ Timer; static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + void SetSignalHandler(const FuzzingOptions& Options) { HandlerOpt = &Options; diff --git a/lib/fuzzer/FuzzerValueBitMap.h b/lib/fuzzer/FuzzerValueBitMap.h index 13d7cbd95..bc039f1df 100644 --- a/lib/fuzzer/FuzzerValueBitMap.h +++ b/lib/fuzzer/FuzzerValueBitMap.h @@ -1,9 +1,8 @@ //===- FuzzerValueBitMap.h - INTERNAL - Bit map -----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // ValueBitMap. @@ -35,7 +34,7 @@ struct ValueBitMap { uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; uintptr_t Old = Map[WordIdx]; - uintptr_t New = Old | (1UL << BitIdx); + uintptr_t New = Old | (1ULL << BitIdx); Map[WordIdx] = New; return New != Old; } @@ -49,7 +48,7 @@ struct ValueBitMap { assert(Idx < kMapSizeInBits); uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; - return Map[WordIdx] & (1UL << BitIdx); + return Map[WordIdx] & (1ULL << BitIdx); } size_t SizeInBits() const { return kMapSizeInBits; } @@ -65,7 +64,7 @@ struct ValueBitMap { } private: - uintptr_t Map[kMapSizeInWords] __attribute__((aligned(512))); + ATTRIBUTE_ALIGNED(512) uintptr_t Map[kMapSizeInWords]; }; } // namespace fuzzer diff --git a/lib/fuzzer/afl/afl_driver.cpp b/lib/fuzzer/afl/afl_driver.cpp index 5a10c0d27..f21dfc58f 100644 --- a/lib/fuzzer/afl/afl_driver.cpp +++ b/lib/fuzzer/afl/afl_driver.cpp @@ -1,9 +1,8 @@ //===- afl_driver.cpp - a glue between AFL and libFuzzer --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// /* This file allows to fuzz libFuzzer-style target functions @@ -32,32 +31,22 @@ clang++ afl_driver.cpp test_fuzzer.o afl-llvm-rt.o.o rm -rf IN OUT; mkdir IN OUT; echo z > IN/z; $AFL_HOME/afl-fuzz -i IN -o OUT ./a.out ################################################################################ -Environment Variables: -There are a few environment variables that can be set to use features that -afl-fuzz doesn't have. - AFL_DRIVER_STDERR_DUPLICATE_FILENAME: Setting this *appends* stderr to the file specified. If the file does not exist, it is created. This is useful for getting -stack traces (when using ASAN for example) or original error messages on hard to -reproduce bugs. +stack traces (when using ASAN for example) or original error messages on hard +to reproduce bugs. Note that any content written to stderr will be written to +this file instead of stderr's usual location. -AFL_DRIVER_EXTRA_STATS_FILENAME: Setting this causes afl_driver to write extra -statistics to the file specified. Currently these are peak_rss_mb -(the peak amount of virtual memory used in MB) and slowest_unit_time_secs. If -the file does not exist it is created. If the file does exist then -afl_driver assumes it was restarted by afl-fuzz and will try to read old -statistics from the file. If that fails then the process will quit. +AFL_DRIVER_CLOSE_FD_MASK: Similar to libFuzzer's -close_fd_mask behavior option. +If 1, close stdout at startup. If 2 close stderr; if 3 close both. */ #include <assert.h> #include <errno.h> -#include <signal.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <sys/resource.h> -#include <sys/time.h> #include <unistd.h> #include <fstream> @@ -99,17 +88,6 @@ statistics from the file. If that fails then the process will quit. #error "Support for your platform has not been implemented" #endif -// Used to avoid repeating error checking boilerplate. If cond is false, a -// fatal error has occurred in the program. In this event print error_message -// to stderr and abort(). Otherwise do nothing. Note that setting -// AFL_DRIVER_STDERR_DUPLICATE_FILENAME may cause error_message to be appended -// to the file as well, if the error occurs after the duplication is performed. -#define CHECK_ERROR(cond, error_message) \ - if (!(cond)) { \ - fprintf(stderr, "%s\n", (error_message)); \ - abort(); \ - } - // libFuzzer interface is thin, so we don't include any libFuzzer headers. extern "C" { int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); @@ -123,24 +101,24 @@ static volatile char suppress_warning2 = AFL_PERSISTENT[0]; // Notify AFL about deferred forkserver. static volatile char AFL_DEFER_FORKSVR[] = "##SIG_AFL_DEFER_FORKSRV##"; -extern "C" void __afl_manual_init(); +extern "C" void __afl_manual_init(); static volatile char suppress_warning1 = AFL_DEFER_FORKSVR[0]; // Input buffer. static const size_t kMaxAflInputSize = 1 << 20; static uint8_t AflInputBuf[kMaxAflInputSize]; -// Variables we need for writing to the extra stats file. -static FILE *extra_stats_file = NULL; -static uint32_t previous_peak_rss = 0; -static time_t slowest_unit_time_secs = 0; -static const int kNumExtraStats = 2; -static const char *kExtraStatsFormatString = "peak_rss_mb : %u\n" - "slowest_unit_time_sec : %u\n"; +// Use this optionally defined function to output sanitizer messages even if +// user asks to close stderr. +__attribute__((weak)) extern "C" void __sanitizer_set_report_fd(void *); + +// Keep track of where stderr content is being written to, so that +// dup_and_close_stderr can use the correct one. +static FILE *output_file = stderr; // Experimental feature to use afl_driver without AFL's deferred mode. // Needs to run before __afl_auto_init. -__attribute__((constructor(0))) void __decide_deferred_forkserver(void) { +__attribute__((constructor(0))) static void __decide_deferred_forkserver(void) { if (getenv("AFL_DRIVER_DONT_DEFER")) { if (unsetenv("__AFL_DEFER_FORKSRV")) { perror("Failed to unset __AFL_DEFER_FORKSRV"); @@ -149,126 +127,15 @@ __attribute__((constructor(0))) void __decide_deferred_forkserver(void) { } } -// Copied from FuzzerUtil.cpp. -size_t GetPeakRSSMb() { - struct rusage usage; - if (getrusage(RUSAGE_SELF, &usage)) - return 0; - if (LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || - LIBFUZZER_OPENBSD) { - // ru_maxrss is in KiB - return usage.ru_maxrss >> 10; - } else if (LIBFUZZER_APPLE) { - // ru_maxrss is in bytes - return usage.ru_maxrss >> 20; - } - assert(0 && "GetPeakRSSMb() is not implemented for your platform"); - return 0; -} - -// Based on SetSigaction in FuzzerUtil.cpp -static void SetSigaction(int signum, - void (*callback)(int, siginfo_t *, void *)) { - struct sigaction sigact; - memset(&sigact, 0, sizeof(sigact)); - sigact.sa_sigaction = callback; - if (sigaction(signum, &sigact, 0)) { - fprintf(stderr, "libFuzzer: sigaction failed with %d\n", errno); - exit(1); - } -} - -// Write extra stats to the file specified by the user. If none is specified -// this function will never be called. -static void write_extra_stats() { - uint32_t peak_rss = GetPeakRSSMb(); - - if (peak_rss < previous_peak_rss) - peak_rss = previous_peak_rss; - - int chars_printed = fprintf(extra_stats_file, kExtraStatsFormatString, - peak_rss, slowest_unit_time_secs); - - CHECK_ERROR(chars_printed != 0, "Failed to write extra_stats_file"); - - CHECK_ERROR(fclose(extra_stats_file) == 0, - "Failed to close extra_stats_file"); -} - -// Call write_extra_stats before we exit. -static void crash_handler(int, siginfo_t *, void *) { - // Make sure we don't try calling write_extra_stats again if we crashed while - // trying to call it. - static bool first_crash = true; - CHECK_ERROR(first_crash, - "Crashed in crash signal handler. This is a bug in the fuzzer."); - - first_crash = false; - write_extra_stats(); -} - -// If the user has specified an extra_stats_file through the environment -// variable AFL_DRIVER_EXTRA_STATS_FILENAME, then perform necessary set up -// to write stats to it on exit. If no file is specified, do nothing. Otherwise -// install signal and exit handlers to write to the file when the process exits. -// Then if the file doesn't exist create it and set extra stats to 0. But if it -// does exist then read the initial values of the extra stats from the file -// and check that the file is writable. -static void maybe_initialize_extra_stats() { - // If AFL_DRIVER_EXTRA_STATS_FILENAME isn't set then we have nothing to do. - char *extra_stats_filename = getenv("AFL_DRIVER_EXTRA_STATS_FILENAME"); - if (!extra_stats_filename) - return; - - // Open the file and find the previous peak_rss_mb value. - // This is necessary because the fuzzing process is restarted after N - // iterations are completed. So we may need to get this value from a previous - // process to be accurate. - extra_stats_file = fopen(extra_stats_filename, "r"); - - // If extra_stats_file already exists: read old stats from it. - if (extra_stats_file) { - int matches = fscanf(extra_stats_file, kExtraStatsFormatString, - &previous_peak_rss, &slowest_unit_time_secs); - - // Make sure we have read a real extra stats file and that we have used it - // to set slowest_unit_time_secs and previous_peak_rss. - CHECK_ERROR(matches == kNumExtraStats, "Extra stats file is corrupt"); - - CHECK_ERROR(fclose(extra_stats_file) == 0, "Failed to close file"); - - // Now open the file for writing. - extra_stats_file = fopen(extra_stats_filename, "w"); - CHECK_ERROR(extra_stats_file, - "Failed to open extra stats file for writing"); - } else { - // Looks like this is the first time in a fuzzing job this is being called. - extra_stats_file = fopen(extra_stats_filename, "w+"); - CHECK_ERROR(extra_stats_file, "failed to create extra stats file"); - } - - // Make sure that crash_handler gets called on any kind of fatal error. - int crash_signals[] = {SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE, SIGINT, - SIGTERM}; - - const size_t num_signals = sizeof(crash_signals) / sizeof(crash_signals[0]); - - for (size_t idx = 0; idx < num_signals; idx++) - SetSigaction(crash_signals[idx], crash_handler); - - // Make sure it gets called on other kinds of exits. - atexit(write_extra_stats); -} - // If the user asks us to duplicate stderr, then do it. static void maybe_duplicate_stderr() { - char* stderr_duplicate_filename = + char *stderr_duplicate_filename = getenv("AFL_DRIVER_STDERR_DUPLICATE_FILENAME"); if (!stderr_duplicate_filename) return; - FILE* stderr_duplicate_stream = + FILE *stderr_duplicate_stream = freopen(stderr_duplicate_filename, "a+", stderr); if (!stderr_duplicate_stream) { @@ -277,6 +144,54 @@ static void maybe_duplicate_stderr() { "Failed to duplicate stderr to AFL_DRIVER_STDERR_DUPLICATE_FILENAME"); abort(); } + output_file = stderr_duplicate_stream; +} + +// Most of these I/O functions were inspired by/copied from libFuzzer's code. +static void discard_output(int fd) { + FILE *temp = fopen("/dev/null", "w"); + if (!temp) + abort(); + dup2(fileno(temp), fd); + fclose(temp); +} + +static void close_stdout() { discard_output(STDOUT_FILENO); } + +// Prevent the targeted code from writing to "stderr" but allow sanitizers and +// this driver to do so. +static void dup_and_close_stderr() { + int output_fileno = fileno(output_file); + int output_fd = dup(output_fileno); + if (output_fd <= 0) + abort(); + FILE *new_output_file = fdopen(output_fd, "w"); + if (!new_output_file) + abort(); + if (!__sanitizer_set_report_fd) + return; + __sanitizer_set_report_fd(reinterpret_cast<void *>(output_fd)); + discard_output(output_fileno); +} + +static void Printf(const char *Fmt, ...) { + va_list ap; + va_start(ap, Fmt); + vfprintf(output_file, Fmt, ap); + va_end(ap); + fflush(output_file); +} + +// Close stdout and/or stderr if user asks for it. +static void maybe_close_fd_mask() { + char *fd_mask_str = getenv("AFL_DRIVER_CLOSE_FD_MASK"); + if (!fd_mask_str) + return; + int fd_mask = atoi(fd_mask_str); + if (fd_mask & 2) + dup_and_close_stderr(); + if (fd_mask & 1) + close_stdout(); } // Define LLVMFuzzerMutate to avoid link failures for targets that use it @@ -287,7 +202,7 @@ extern "C" size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { } // Execute any files provided as parameters. -int ExecuteFilesOnyByOne(int argc, char **argv) { +static int ExecuteFilesOnyByOne(int argc, char **argv) { for (int i = 1; i < argc; i++) { std::ifstream in(argv[i], std::ios::binary); in.seekg(0, in.end); @@ -306,7 +221,7 @@ int ExecuteFilesOnyByOne(int argc, char **argv) { } int main(int argc, char **argv) { - fprintf(stderr, + Printf( "======================= INFO =========================\n" "This binary is built for AFL-fuzz.\n" "To run the target function on individual input(s) execute this:\n" @@ -319,13 +234,13 @@ int main(int argc, char **argv) { "re-spawning the process (default: 1000)\n" "======================================================\n", argv[0], argv[0], argv[0]); + + maybe_duplicate_stderr(); + maybe_close_fd_mask(); if (LLVMFuzzerInitialize) LLVMFuzzerInitialize(&argc, &argv); // Do any other expensive one-time initialization here. - maybe_duplicate_stderr(); - maybe_initialize_extra_stats(); - if (!getenv("AFL_DRIVER_DONT_DEFER")) __afl_manual_init(); @@ -333,8 +248,7 @@ int main(int argc, char **argv) { if (argc == 2 && argv[1][0] == '-') N = atoi(argv[1] + 1); else if(argc == 2 && (N = atoi(argv[1])) > 0) - fprintf(stderr, "WARNING: using the deprecated call style `%s %d`\n", - argv[0], N); + Printf("WARNING: using the deprecated call style `%s %d`\n", argv[0], N); else if (argc > 1) return ExecuteFilesOnyByOne(argc, argv); @@ -345,7 +259,6 @@ int main(int argc, char **argv) { uint8_t dummy_input[1] = {0}; LLVMFuzzerTestOneInput(dummy_input, 1); - time_t unit_time_secs; int num_runs = 0; while (__afl_persistent_loop(N)) { ssize_t n_read = read(0, AflInputBuf, kMaxAflInputSize); @@ -354,25 +267,10 @@ int main(int argc, char **argv) { // overflows. Don't use unique_ptr/etc to avoid extra dependencies. uint8_t *copy = new uint8_t[n_read]; memcpy(copy, AflInputBuf, n_read); - - struct timeval unit_start_time; - CHECK_ERROR(gettimeofday(&unit_start_time, NULL) == 0, - "Calling gettimeofday failed"); - num_runs++; LLVMFuzzerTestOneInput(copy, n_read); - - struct timeval unit_stop_time; - CHECK_ERROR(gettimeofday(&unit_stop_time, NULL) == 0, - "Calling gettimeofday failed"); - - // Update slowest_unit_time_secs if we see a new max. - unit_time_secs = unit_stop_time.tv_sec - unit_start_time.tv_sec; - if (slowest_unit_time_secs < unit_time_secs) - slowest_unit_time_secs = unit_time_secs; - delete[] copy; } } - fprintf(stderr, "%s: successfully executed %d input(s)\n", argv[0], num_runs); + Printf("%s: successfully executed %d input(s)\n", argv[0], num_runs); } diff --git a/lib/fuzzer/dataflow/DataFlow.cpp b/lib/fuzzer/dataflow/DataFlow.cpp index a79c796ac..187a8e52c 100644 --- a/lib/fuzzer/dataflow/DataFlow.cpp +++ b/lib/fuzzer/dataflow/DataFlow.cpp @@ -1,9 +1,8 @@ /*===- DataFlow.cpp - a standalone DataFlow tracer -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // An experimental data-flow tracer for fuzz targets. @@ -64,6 +63,9 @@ __attribute__((weak)) extern int LLVMFuzzerInitialize(int *argc, char ***argv); } // extern "C" static size_t InputLen; +static size_t InputLabelBeg; +static size_t InputLabelEnd; +static size_t InputSizeLabel; static size_t NumFuncs; static const uintptr_t *FuncsBeg; static __thread size_t CurrentFunc; @@ -96,8 +98,10 @@ void SetBytesForLabel(dfsan_label L, char *Bytes) { return; LabelSeen[L] = true; assert(L); - if (L <= InputLen + 1) { - Bytes[L - 1] = '1'; + if (L < InputSizeLabel) { + Bytes[L + InputLabelBeg - 1] = '1'; + } else if (L == InputSizeLabel) { + Bytes[InputLen] = '1'; } else { auto *DLI = dfsan_get_label_info(L); SetBytesForLabel(DLI->l1, Bytes); @@ -125,9 +129,9 @@ int main(int argc, char **argv) { if (argc == 1) return PrintFunctions(); assert(argc == 4 || argc == 5); - size_t Beg = atoi(argv[1]); - size_t End = atoi(argv[2]); - assert(Beg < End); + InputLabelBeg = atoi(argv[1]); + InputLabelEnd = atoi(argv[2]); + assert(InputLabelBeg < InputLabelEnd); const char *Input = argv[3]; fprintf(stderr, "INFO: reading '%s'\n", Input); @@ -144,14 +148,16 @@ int main(int argc, char **argv) { fprintf(stderr, "INFO: running '%s'\n", Input); for (size_t I = 1; I <= InputLen; I++) { - dfsan_label L = dfsan_create_label("", nullptr); - assert(L == I); size_t Idx = I - 1; - if (Idx >= Beg && Idx < End) + if (Idx >= InputLabelBeg && Idx < InputLabelEnd) { + dfsan_label L = dfsan_create_label("", nullptr); + assert(L == I - InputLabelBeg); dfsan_set_label(L, Buf + Idx, 1); + } } dfsan_label SizeL = dfsan_create_label("", nullptr); - assert(SizeL == InputLen + 1); + InputSizeLabel = SizeL; + assert(InputSizeLabel == InputLabelEnd - InputLabelBeg + 1); dfsan_set_label(SizeL, &InputLen, sizeof(InputLen)); LLVMFuzzerTestOneInput(Buf, InputLen); diff --git a/lib/fuzzer/scripts/collect_data_flow.py b/lib/fuzzer/scripts/collect_data_flow.py index 3edff66bb..bd601eb63 100755 --- a/lib/fuzzer/scripts/collect_data_flow.py +++ b/lib/fuzzer/scripts/collect_data_flow.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #===- lib/fuzzer/scripts/collect_data_flow.py ------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # Runs the data-flow tracer several times on the same input in order to collect @@ -40,7 +39,9 @@ def collect_dataflow_for_corpus(self, exe, corpus_dir, output_dir): for root, dirs, files in os.walk(corpus_dir): for f in files: path = os.path.join(root, f) - sha1 = hashlib.sha1(open(path).read()).hexdigest() + with open(path, 'rb') as fh: + data = fh.read() + sha1 = hashlib.sha1(data).hexdigest() output = os.path.join(output_dir, sha1) subprocess.call([self, exe, path, output]) functions_txt = open(os.path.join(output_dir, "functions.txt"), "w") @@ -56,19 +57,19 @@ def main(argv): q = [[0, size]] tmpdir = tempfile.mkdtemp(prefix="libfuzzer-tmp-") atexit.register(cleanup, tmpdir) - print "tmpdir: ", tmpdir + print("tmpdir: ", tmpdir) outputs = [] while len(q): r = q.pop() - print "******* Trying: ", r + print("******* Trying: ", r) tmpfile = os.path.join(tmpdir, str(r[0]) + "-" + str(r[1])) ret = subprocess.call([exe, str(r[0]), str(r[1]), inp, tmpfile]) if ret and r[1] - r[0] >= 2: - q.append([r[0], (r[1] + r[0]) / 2]) - q.append([(r[1] + r[0]) / 2, r[1]]) + q.append([r[0], (r[1] + r[0]) // 2]) + q.append([(r[1] + r[0]) // 2, r[1]]) else: outputs.append(tmpfile) - print "******* Success: ", r + print("******* Success: ", r) f = sys.stdout if len(argv) >= 4: f = open(argv[3], "w") diff --git a/lib/fuzzer/scripts/merge_data_flow.py b/lib/fuzzer/scripts/merge_data_flow.py index d2f5081e7..b442b89b7 100755 --- a/lib/fuzzer/scripts/merge_data_flow.py +++ b/lib/fuzzer/scripts/merge_data_flow.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #===- lib/fuzzer/scripts/merge_data_flow.py ------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # Merge several data flow traces into one. @@ -19,7 +18,7 @@ def Merge(a, b): res = array('b') for i in range(0, len(a)): res.append(ord('1' if a[i] == '1' or b[i] == '1' else '0')) - return res.tostring() + return res.tostring().decode('utf-8') def main(argv): D = {} @@ -30,7 +29,11 @@ def main(argv): else: D[F] = BV; for F in D.keys(): - print("%s %s" % (F, D[F])) + if isinstance(D[F], str): + value = D[F] + else: + value = D[F].decode('utf-8') + print("%s %s" % (F, value)) if __name__ == '__main__': main(sys.argv) diff --git a/lib/fuzzer/scripts/unbalanced_allocs.py b/lib/fuzzer/scripts/unbalanced_allocs.py index 74478ad55..579e481a2 100755 --- a/lib/fuzzer/scripts/unbalanced_allocs.py +++ b/lib/fuzzer/scripts/unbalanced_allocs.py @@ -1,10 +1,9 @@ #!/usr/bin/env python #===- lib/fuzzer/scripts/unbalanced_allocs.py ------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # diff --git a/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c b/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c index 0d76ea49e..efe512cfe 100644 --- a/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c +++ b/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c @@ -1,9 +1,8 @@ /*===- StandaloneFuzzTargetMain.c - standalone main() for fuzz targets. ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This main() function can be linked to a fuzz target (i.e. a library @@ -33,6 +32,7 @@ int main(int argc, char **argv) { fseek(f, 0, SEEK_SET); unsigned char *buf = (unsigned char*)malloc(len); size_t n_read = fread(buf, 1, len, f); + fclose(f); assert(n_read == len); LLVMFuzzerTestOneInput(buf, len); free(buf); diff --git a/lib/fuzzer/tests/CMakeLists.txt b/lib/fuzzer/tests/CMakeLists.txt index 6abb72def..f338d9993 100644 --- a/lib/fuzzer/tests/CMakeLists.txt +++ b/lib/fuzzer/tests/CMakeLists.txt @@ -17,16 +17,21 @@ set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "Compiler-RT Tests") set(LIBFUZZER_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++) -if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") - list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lc++ -lpthread) -elseif(NOT WIN32) - list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lstdc++ -lpthread) +if(NOT WIN32) + list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lpthread) endif() -if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++) endif() +if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS) + # Match visibility settings. + list(APPEND LIBFUZZER_UNITTEST_CFLAGS "-fvisibility=hidden") +endif() + if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) # libFuzzer unit tests are only run on the host machine. set(arch ${COMPILER_RT_DEFAULT_TARGET_ARCH}) @@ -45,8 +50,11 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} FOLDER "Compiler-RT Runtime tests") - if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH) - set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch}-build) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) + file(GLOB libfuzzer_headers ../*.h) + set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch}-build ${libfuzzer_headers}) set(LIBFUZZER_TEST_RUNTIME_CFLAGS -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) set(LIBFUZZER_TEST_RUNTIME_LINK_FLAGS ${LIBCXX_${arch}_PREFIX}/lib/libc++.a) endif() diff --git a/lib/fuzzer/tests/FuzzerUnittest.cpp b/lib/fuzzer/tests/FuzzerUnittest.cpp index 7cdd44582..d9bdfc1ef 100644 --- a/lib/fuzzer/tests/FuzzerUnittest.cpp +++ b/lib/fuzzer/tests/FuzzerUnittest.cpp @@ -1,5 +1,6 @@ -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // Avoid ODR violations (LibFuzzer is built without ASan and this test is built // with ASan) involving C++ standard library types when using libcxx. @@ -618,7 +619,7 @@ TEST(Merge, Bad) { "2\n2\nA\n", "2\n2\nA\nB\nC\n", "0\n0\n", - "1\n1\nA\nDONE 0", + "1\n1\nA\nFT 0", "1\n1\nA\nSTARTED 1", }; Merger M; @@ -643,11 +644,9 @@ static void Merge(const std::string &Input, size_t NumNewFeatures) { Merger M; Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; EXPECT_TRUE(M.Parse(Input, true)); - std::stringstream SS; - M.PrintSummary(SS); - EXPECT_EQ(NumNewFeatures, M.Merge(&NewFiles)); - EXPECT_EQ(M.AllFeatures(), M.ParseSummary(SS)); + EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, Result); } @@ -671,9 +670,9 @@ TEST(Merge, Good) { EXPECT_TRUE(M.Parse("3\n1\nAA\nBB\nC\n" "STARTED 0 1000\n" - "DONE 0 1 2 3\n" + "FT 0 1 2 3\n" "STARTED 1 1001\n" - "DONE 1 4 5 6 \n" + "FT 1 4 5 6 \n" "STARTED 2 1002\n" "", true)); EXPECT_EQ(M.Files.size(), 3U); @@ -691,11 +690,12 @@ TEST(Merge, Good) { Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n" "", true)); EXPECT_EQ(M.Files.size(), 3U); EXPECT_EQ(M.NumFilesInFirstCorpus, 2U); @@ -704,24 +704,24 @@ TEST(Merge, Good) { EQ(M.Files[0].Features, {1, 2, 3}); EQ(M.Files[1].Features, {4, 5, 6}); EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(0U, M.Merge(&NewFiles)); + EXPECT_EQ(0U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {}); EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3\n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3\n" "", true)); EQ(M.Files[0].Features, {1, 2, 3}); EQ(M.Files[1].Features, {4, 5, 6}); EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(3U, M.Merge(&NewFiles)); + EXPECT_EQ(3U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {"B"}); // Same as the above, but with InitialFeatures. EXPECT_TRUE(M.Parse("2\n0\nB\nC\n" - "STARTED 0 1001\nDONE 0 4 5 6 \n" - "STARTED 1 1002\nDONE 1 6 1 3\n" + "STARTED 0 1001\nFT 0 4 5 6 \n" + "STARTED 1 1002\nFT 1 6 1 3\n" "", true)); EQ(M.Files[0].Features, {4, 5, 6}); EQ(M.Files[1].Features, {1, 3, 6}); @@ -729,36 +729,36 @@ TEST(Merge, Good) { InitialFeatures.insert(1); InitialFeatures.insert(2); InitialFeatures.insert(3); - EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFiles)); + EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {"B"}); } TEST(Merge, Merge) { Merge("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"B"}, 3); Merge("3\n0\nA\nB\nC\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"A", "B", "C"}, 6); Merge("4\n0\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1101\nDONE 1 4 5 6 \n" - "STARTED 2 1102\nDONE 2 6 1 3 100 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1101\nFT 1 4 5 6 \n" + "STARTED 2 1102\nFT 2 6 1 3 100 \n" + "STARTED 3 1000\nFT 3 1 \n", {"A", "B", "C", "D"}, 7); Merge("4\n1\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 4 5 6 7 8\n" - "STARTED 1 1100\nDONE 1 1 2 3 \n" - "STARTED 2 1100\nDONE 2 2 3 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 4 5 6 7 8\n" + "STARTED 1 1100\nFT 1 1 2 3 \n" + "STARTED 2 1100\nFT 2 2 3 \n" + "STARTED 3 1000\nFT 3 1 \n", {"B", "D"}, 3); } diff --git a/lib/hwasan/CMakeLists.txt b/lib/hwasan/CMakeLists.txt index 20ab94dc0..8fa59199e 100644 --- a/lib/hwasan/CMakeLists.txt +++ b/lib/hwasan/CMakeLists.txt @@ -2,20 +2,23 @@ include_directories(..) # Runtime library sources and build flags. set(HWASAN_RTL_SOURCES - hwasan.cc - hwasan_allocator.cc - hwasan_dynamic_shadow.cc - hwasan_interceptors.cc - hwasan_linux.cc - hwasan_memintrinsics.cc - hwasan_poisoning.cc - hwasan_report.cc - hwasan_thread.cc - hwasan_thread_list.cc + hwasan.cpp + hwasan_allocator.cpp + hwasan_dynamic_shadow.cpp + hwasan_interceptors.cpp + hwasan_interceptors_vfork.S + hwasan_linux.cpp + hwasan_memintrinsics.cpp + hwasan_poisoning.cpp + hwasan_report.cpp + hwasan_tag_mismatch_aarch64.S + hwasan_thread.cpp + hwasan_thread_list.cpp ) set(HWASAN_RTL_CXX_SOURCES - hwasan_new_delete.cc) + hwasan_new_delete.cpp + ) set(HWASAN_RTL_HEADERS hwasan.h @@ -24,6 +27,7 @@ set(HWASAN_RTL_HEADERS hwasan_flags.h hwasan_flags.inc hwasan_interface_internal.h + hwasan_malloc_bisect.h hwasan_mapping.h hwasan_poisoning.h hwasan_report.h @@ -55,7 +59,7 @@ append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC -ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS) append_list_if(MSVC /DEBUG HWASAN_DYNAMIC_LINK_FLAGS) -set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl HWASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBRT rt HWASAN_DYNAMIC_LIBS) diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cpp index e2bfea5e4..65b755ee2 100644 --- a/lib/hwasan/hwasan.cc +++ b/lib/hwasan/hwasan.cpp @@ -1,9 +1,8 @@ -//===-- hwasan.cc ---------------------------------------------------------===// +//===-- hwasan.cpp --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "hwasan.h" #include "hwasan_checks.h" +#include "hwasan_dynamic_shadow.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" @@ -58,7 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; -int hwasan_shadow_inited = 0; +int hwasan_instrumentation_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -88,6 +88,8 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // 8 shadow pages ~512kB, small enough to cover common stack sizes. + cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; @@ -142,23 +144,6 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - Thread *t = GetCurrentThread(); - if (!t) { - // the thread is still being created. - stack->size = 0; - return; - } - if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -188,17 +173,13 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { #if SANITIZER_ANDROID static char *memory_usage_buffer = nullptr; -#define PR_SET_VMA 0x53564d41 -#define PR_SET_VMA_ANON_NAME 0 - static void InitMemoryUsage() { memory_usage_buffer = (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); CHECK(memory_usage_buffer); memory_usage_buffer[0] = '\0'; - CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, - (uptr)memory_usage_buffer, kMemoryUsageBufferSize, - (uptr)memory_usage_buffer) == 0); + DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, + memory_usage_buffer); } void UpdateMemoryUsage() { @@ -245,28 +226,59 @@ const char *GetStackFrameDescr(uptr pc) { return nullptr; } -} // namespace __hwasan - -// Interface. +// Prepare to run instrumented code on the main thread. +void InitInstrumentation() { + if (hwasan_instrumentation_inited) return; -using namespace __hwasan; - -uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. - -void __hwasan_shadow_init() { - if (hwasan_shadow_inited) return; if (!InitShadow()) { Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); DumpProcessMap(); Die(); } - hwasan_shadow_inited = 1; + + InitThreads(); + hwasanThreadList().CreateCurrentThread(); + + hwasan_instrumentation_inited = 1; } +} // namespace __hwasan + +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __hwasan; + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, request_fast); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + +// Interface. + +using namespace __hwasan; + +uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. + void __hwasan_init_frames(uptr beg, uptr end) { InitFrameDescriptors(beg, end); } +void __hwasan_init_static() { + InitShadowGOT(); + InitInstrumentation(); +} + void __hwasan_init() { CHECK(!hwasan_init_is_running); if (hwasan_inited) return; @@ -287,10 +299,11 @@ void __hwasan_init() { DisableCoreDumperIfNecessary(); - __hwasan_shadow_init(); + InitInstrumentation(); - InitThreads(); - hwasanThreadList().CreateCurrentThread(); + // Needs to be called here because flags()->random_tags might not have been + // initialized when InitInstrumentation() was called. + GetCurrentThread()->InitRandomState(); MadviseShadow(); @@ -335,14 +348,14 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { if (sz == 0) return -1; tag_t ptr_tag = GetTagFromPointer((uptr)p); - if (ptr_tag == 0) - return -1; uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); uptr shadow_first = MemToShadow(ptr_raw); uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) - if (*(tag_t*)s != ptr_tag) - return ShadowToMem(s) - ptr_raw; + if (*(tag_t *)s != ptr_tag) { + sptr offset = ShadowToMem(s) - ptr_raw; + return offset < 0 ? 0 : offset; + } return -1; } @@ -467,6 +480,28 @@ void __hwasan_handle_longjmp(const void *sp_dst) { TagMemory(sp, dst - sp, 0); } +void __hwasan_handle_vfork(const void *sp_dst) { + uptr sp = (uptr)sp_dst; + Thread *t = GetCurrentThread(); + CHECK(t); + uptr top = t->stack_top(); + uptr bottom = t->stack_bottom(); + if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " + "stack top: %zx; current %zx; bottom: %zx \n" + "False positive error reports may follow\n", + top, sp, bottom); + return; + } + TagMemory(bottom, sp - bottom, 0); +} + +extern "C" void *__hwasan_extra_spill_area() { + Thread *t = GetCurrentThread(); + return &t->vfork_spill(); +} + void __hwasan_print_memory_usage() { InternalScopedString s(kMemoryUsageBufferSize); HwasanFormatMemoryUsage(s); diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h index ce9e904c5..9cc9490a9 100644 --- a/lib/hwasan/hwasan.h +++ b/lib/hwasan/hwasan.h @@ -1,9 +1,8 @@ //===-- hwasan.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,6 +70,7 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); void InitThreads(); +void InitInstrumentation(); void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); @@ -81,6 +81,7 @@ void HwasanAllocatorThreadFinish(); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *hwasan_valloc(uptr size, StackTrace *stack); void *hwasan_pvalloc(uptr size, StackTrace *stack); void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -104,9 +105,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Returns a "chained" origin id, pointing to the given stack trace followed by // the previous origin id. u32 ChainOrigin(u32 id, StackTrace *stack); @@ -115,16 +113,15 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + if (hwasan_inited) \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + if (hwasan_inited) \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cpp index 8487ed7e1..fd5248796 100644 --- a/lib/hwasan/hwasan_allocator.cc +++ b/lib/hwasan/hwasan_allocator.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_allocator.cc ------------------------- ---------------------===// +//===-- hwasan_allocator.cpp ------------------------ ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_malloc_bisect.h" #include "hwasan_thread.h" #include "hwasan_report.h" @@ -177,10 +177,16 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, size - orig_size); void *user_ptr = allocated; - if (flags()->tag_in_malloc && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - user_ptr = (void *)TagMemoryAligned( - (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); + // Tagging can only be skipped when both tag_in_malloc and tag_in_free are + // false. When tag_in_malloc = false and tag_in_free = true malloc needs to + // retag to 0. + if ((flags()->tag_in_malloc || flags()->tag_in_free) && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + tag_t tag = flags()->tag_in_malloc && malloc_bisect(stack, orig_size) + ? (t ? t->GenerateRandomTag() : kFallbackAllocTag) + : 0; + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, tag); + } if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && right_align_mode) { @@ -242,7 +248,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } - if (flags()->tag_in_free && + if (flags()->tag_in_free && malloc_bisect(stack, 0) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); @@ -335,6 +341,16 @@ void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); } +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return hwasan_realloc(ptr, nmemb * size, stack); +} + void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( HwasanAllocate(stack, size, GetPageSizeCached(), false)); diff --git a/lib/hwasan/hwasan_allocator.h b/lib/hwasan/hwasan_allocator.h index 6ab722fa6..3a50a11f3 100644 --- a/lib/hwasan/hwasan_allocator.h +++ b/lib/hwasan/hwasan_allocator.h @@ -1,9 +1,8 @@ //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,10 +61,8 @@ struct AP64 { static const uptr kFlags = 0; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); diff --git a/lib/hwasan/hwasan_checks.h b/lib/hwasan/hwasan_checks.h index 688b5e2be..693faa0c2 100644 --- a/lib/hwasan/hwasan_checks.h +++ b/lib/hwasan/hwasan_checks.h @@ -1,9 +1,8 @@ //===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define HWASAN_CHECKS_H #include "hwasan_mapping.h" +#include "sanitizer_common/sanitizer_common.h" namespace __hwasan { template <unsigned X> @@ -23,8 +23,8 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { (void)p; // 0x900 is added to do not interfere with the kernel use of lower values of // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); + register uptr x0 asm("x0") = p; + asm("brk %1\n\t" ::"r"(x0), "n"(0x900 + X)); #elif defined(__x86_64__) // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes // total. The pointer is passed via rdi. @@ -42,6 +42,25 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { // __builtin_unreachable(); } +// Version with access size which is not power of 2 +template <unsigned X> +__attribute__((always_inline)) static void SigTrap(uptr p, uptr size) { +#if defined(__aarch64__) + register uptr x0 asm("x0") = p; + register uptr x1 asm("x1") = size; + asm("brk %2\n\t" ::"r"(x0), "r"(x1), "n"(0x900 + X)); +#elif defined(__x86_64__) + // Size is stored in rsi. + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p), "S"(size)); +#else + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + enum class ErrorAction { Abort, Recover }; enum class AccessType { Load, Store }; @@ -70,7 +89,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, for (tag_t *t = shadow_first; t <= shadow_last; ++t) if (UNLIKELY(ptr_tag != *t)) { SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); if (EA == ErrorAction::Abort) __builtin_unreachable(); } diff --git a/lib/hwasan/hwasan_dynamic_shadow.cc b/lib/hwasan/hwasan_dynamic_shadow.cpp index 87670f508..a04751f44 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===// +//===-- hwasan_dynamic_shadow.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,6 +18,9 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_posix.h" +#include <elf.h> +#include <link.h> + // The code in this file needs to run in an unrelocated binary. It should not // access any external symbol, including its own non-hidden globals. @@ -119,10 +121,28 @@ decltype(__hwasan_shadow)* __hwasan_premap_shadow() { INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow"))) void __hwasan_shadow(); +extern __attribute((weak, visibility("hidden"))) ElfW(Rela) __rela_iplt_start[], + __rela_iplt_end[]; + } // extern "C" namespace __hwasan { +void InitShadowGOT() { + // Call the ifunc resolver for __hwasan_shadow and fill in its GOT entry. This + // needs to be done before other ifunc resolvers (which are handled by libc) + // because a resolver might read __hwasan_shadow. + typedef ElfW(Addr) (*ifunc_resolver_t)(void); + for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) { + ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset); + ElfW(Addr) resolver = r->r_addend; + if (resolver == reinterpret_cast<ElfW(Addr)>(&__hwasan_premap_shadow)) { + *offset = reinterpret_cast<ifunc_resolver_t>(resolver)(); + break; + } + } +} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); @@ -133,10 +153,12 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) { #else namespace __hwasan { +void InitShadowGOT() {} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan -# + #endif // SANITIZER_ANDROID diff --git a/lib/hwasan/hwasan_dynamic_shadow.h b/lib/hwasan/hwasan_dynamic_shadow.h index b5e9e1dd6..3c2e7c716 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.h +++ b/lib/hwasan/hwasan_dynamic_shadow.h @@ -1,9 +1,8 @@ //===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -21,6 +20,7 @@ namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes); +void InitShadowGOT(); } // namespace __hwasan diff --git a/lib/hwasan/hwasan_flags.h b/lib/hwasan/hwasan_flags.h index 492d5bb98..0a6998f67 100644 --- a/lib/hwasan/hwasan_flags.h +++ b/lib/hwasan/hwasan_flags.h @@ -1,9 +1,8 @@ //===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc index b450ab950..01fdad87a 100644 --- a/lib/hwasan/hwasan_flags.inc +++ b/lib/hwasan/hwasan_flags.inc @@ -1,9 +1,8 @@ //===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,3 +85,16 @@ HWASAN_FLAG(int, stack_history_size, 1024, "The number of stack frames remembered per thread. " "Affects the quality of stack-related reports, but not the ability " "to find bugs.") + +// Malloc / free bisection. Only tag malloc and free calls when a hash of +// allocation size and stack trace is between malloc_bisect_left and +// malloc_bisect_right (both inclusive). [0, 0] range is special and disables +// bisection (i.e. everything is tagged). Once the range is narrowed down +// enough, use malloc_bisect_dump to see interesting allocations. +HWASAN_FLAG(uptr, malloc_bisect_left, 0, + "Left bound of malloc bisection, inclusive.") +HWASAN_FLAG(uptr, malloc_bisect_right, 0, + "Right bound of malloc bisection, inclusive.") +HWASAN_FLAG(bool, malloc_bisect_dump, false, + "Print all allocations within [malloc_bisect_left, " + "malloc_bisect_right] range ") diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cpp index fb0dcb890..e305f6b1b 100644 --- a/lib/hwasan/hwasan_interceptors.cc +++ b/lib/hwasan/hwasan_interceptors.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_interceptors.cc --------------------------------------------===// +//===-- hwasan_interceptors.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -179,6 +178,11 @@ void * __sanitizer_realloc(void *ptr, uptr size) { return hwasan_realloc(ptr, size, &stack); } +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) @@ -205,6 +209,7 @@ INTERCEPTOR_ALIAS(void, free, void *ptr); INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD @@ -228,6 +233,11 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, } #endif +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(int, vfork); +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork); +#endif + static void BeforeFork() { StackDepotLockAll(); } @@ -267,9 +277,12 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(fork); #if HWASAN_WITH_INTERCEPTORS +#if defined(__linux__) + INTERCEPT_FUNCTION(vfork); +#endif // __linux__ #if !defined(__aarch64__) INTERCEPT_FUNCTION(pthread_create); -#endif +#endif // __aarch64__ INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); #endif diff --git a/lib/hwasan/hwasan_interceptors_vfork.S b/lib/hwasan/hwasan_interceptors_vfork.S new file mode 100644 index 000000000..13d0829c0 --- /dev/null +++ b/lib/hwasan/hwasan_interceptors_vfork.S @@ -0,0 +1,10 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS +#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h index d3b2087d0..1b10d76c7 100644 --- a/lib/hwasan/hwasan_interface_internal.h +++ b/lib/hwasan/hwasan_interface_internal.h @@ -1,9 +1,8 @@ //===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,7 +20,7 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -void __hwasan_shadow_init(); +void __hwasan_init_static(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); @@ -101,6 +100,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_mismatch(uptr addr, u8 ts); + +SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); // Returns the offset of the first tag mismatch or -1 if the whole range is @@ -118,6 +120,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_handle_longjmp(const void *sp_dst); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_vfork(const void *sp_dst); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); SANITIZER_INTERFACE_ATTRIBUTE @@ -193,6 +198,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_realloc(void *ptr, uptr size); SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_malloc(uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cpp index 5b0a8b4ac..d93297648 100644 --- a/lib/hwasan/hwasan_linux.cc +++ b/lib/hwasan/hwasan_linux.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===// +//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -39,7 +38,17 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" -#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. +// +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF +// Not currently tested. +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON +// Integration tests downstream exist. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF +// Tested with check-hwasan on x86_64-linux. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON +// Tested with check-hwasan on aarch64-linux-android. +#if !SANITIZER_ANDROID SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL uptr __hwasan_tls; #endif @@ -219,6 +228,8 @@ bool MemIsApp(uptr p) { } static void HwasanAtExit(void) { + if (common_flags()->print_module_map) + DumpProcessMap(); if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) ReportStats(); if (hwasan_report_count > 0) { @@ -235,7 +246,7 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 extern "C" void __hwasan_thread_enter() { - hwasanThreadList().CreateCurrentThread(); + hwasanThreadList().CreateCurrentThread()->InitRandomState(); } extern "C" void __hwasan_thread_exit() { @@ -288,7 +299,9 @@ uptr *GetCurrentThreadLongPtr() { #if SANITIZER_ANDROID void AndroidTestTlsSlot() { uptr kMagicValue = 0x010203040A0B0C0D; - *(uptr *)get_android_tls_ptr() = kMagicValue; + uptr *tls_ptr = GetCurrentThreadLongPtr(); + uptr old_value = *tls_ptr; + *tls_ptr = kMagicValue; dlerror(); if (*(uptr *)get_android_tls_ptr() != kMagicValue) { Printf( @@ -296,6 +309,7 @@ void AndroidTestTlsSlot() { "for dlerror().\n"); Die(); } + *tls_ptr = old_value; } #else void AndroidTestTlsSlot() {} @@ -369,22 +383,35 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } +static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, + ucontext_t *uc, uptr *registers_frame = nullptr) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); + + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { AccessInfo ai = GetAccessInfo(info, uc); if (!ai.is_store && !ai.is_load) return false; - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, uc, common_flags()->fast_unwind_on_fatal); - - ++hwasan_report_count; - - bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); + HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -395,10 +422,25 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { return true; } +// Entry point stub for interoperability between __hwasan_tag_mismatch (ASM) and +// the rest of the mismatch handling code (C++). +extern "C" void __hwasan_tag_mismatch_stub(uptr addr, uptr access_info, + uptr *registers_frame) { + AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.recover = false; + ai.addr = addr; + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); + stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/lib/hwasan/hwasan_malloc_bisect.h b/lib/hwasan/hwasan_malloc_bisect.h new file mode 100644 index 000000000..eaf124aab --- /dev/null +++ b/lib/hwasan/hwasan_malloc_bisect.h @@ -0,0 +1,50 @@ +//===-- hwasan_malloc_bisect.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_hash.h" +#include "hwasan.h" + +namespace __hwasan { + +static u32 malloc_hash(StackTrace *stack, uptr orig_size) { + uptr len = Min(stack->size, (unsigned)7); + MurMur2HashBuilder H(len); + H.add(orig_size); + // Start with frame #1 to skip __sanitizer_malloc frame, which is + // (a) almost always the same (well, could be operator new or new[]) + // (b) can change hashes when compiler-rt is rebuilt, invalidating previous + // bisection results. + // Because of ASLR, use only offset inside the page. + for (uptr i = 1; i < len; ++i) H.add(((u32)stack->trace[i]) & 0xFFF); + return H.get(); +} + +static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) { + uptr left = flags()->malloc_bisect_left; + uptr right = flags()->malloc_bisect_right; + if (LIKELY(left == 0 && right == 0)) + return true; + if (!stack) + return true; + // Allow malloc_bisect_right > (u32)(-1) to avoid spelling the latter in + // decimal. + uptr h = (uptr)malloc_hash(stack, orig_size); + if (h < left || h > right) + return false; + if (flags()->malloc_bisect_dump) { + Printf("[alloc] %u %zu\n", h, orig_size); + stack->Print(); + } + return true; +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_mapping.h b/lib/hwasan/hwasan_mapping.h index e5e23dc60..a86ad7ca8 100644 --- a/lib/hwasan/hwasan_mapping.h +++ b/lib/hwasan/hwasan_mapping.h @@ -1,9 +1,8 @@ //===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_memintrinsics.cc b/lib/hwasan/hwasan_memintrinsics.cpp index 9cb844e45..e82d77a1b 100644 --- a/lib/hwasan/hwasan_memintrinsics.cc +++ b/lib/hwasan/hwasan_memintrinsics.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +//===-- hwasan_memintrinsics.cpp --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cpp index f2e8faf5d..438a3699a 100644 --- a/lib/hwasan/hwasan_new_delete.cc +++ b/lib/hwasan/hwasan_new_delete.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_new_delete.cc ----------------------------------------------===// +//===-- hwasan_new_delete.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc deleted file mode 100644 index 9c8e16b12..000000000 --- a/lib/hwasan/hwasan_poisoning.cc +++ /dev/null @@ -1,37 +0,0 @@ -//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of HWAddressSanitizer. -// -//===----------------------------------------------------------------------===// - -#include "hwasan_poisoning.h" - -#include "hwasan_mapping.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" - -namespace __hwasan { - -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - internal_memset((void *)shadow_start, tag, shadow_size); - return AddTagToPointer(p, tag); -} - -uptr TagMemory(uptr p, uptr size, tag_t tag) { - uptr start = RoundDownTo(p, kShadowAlignment); - uptr end = RoundUpTo(p + size, kShadowAlignment); - return TagMemoryAligned(start, end - start, tag); -} - -} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.cpp b/lib/hwasan/hwasan_poisoning.cpp new file mode 100644 index 000000000..2a0816428 --- /dev/null +++ b/lib/hwasan/hwasan_poisoning.cpp @@ -0,0 +1,52 @@ +//===-- hwasan_poisoning.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "hwasan_mapping.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_linux.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.h b/lib/hwasan/hwasan_poisoning.h index 0dbf9d8ed..61751f7d2 100644 --- a/lib/hwasan/hwasan_poisoning.h +++ b/lib/hwasan/hwasan_poisoning.h @@ -1,9 +1,8 @@ //===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cpp index ea3e4096d..fa2fff742 100644 --- a/lib/hwasan/hwasan_report.cc +++ b/lib/hwasan/hwasan_report.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_report.cc --------------------------------------------------===// +//===-- hwasan_report.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_report.h" #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -35,15 +35,21 @@ class ScopedReport { ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { BlockingMutexLock lock(&error_message_lock_); error_message_ptr_ = fatal ? &error_message_ : nullptr; + ++hwasan_report_count; } ~ScopedReport() { - BlockingMutexLock lock(&error_message_lock_); - if (fatal) { - SetAbortMessage(error_message_.data()); - Die(); + { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) + SetAbortMessage(error_message_.data()); + error_message_ptr_ = nullptr; } - error_message_ptr_ = nullptr; + if (common_flags()->print_module_map >= 2 || + (fatal && common_flags()->print_module_map)) + DumpProcessMap(); + if (fatal) + Die(); } static void MaybeAppendToErrorMessage(const char *msg) { @@ -247,8 +253,8 @@ void PrintAddressDescription( uptr pc_mask = (1ULL << 48) - 1; uptr pc = record & pc_mask; if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); - RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, + frame_desc.append(" sp: 0x%zx ", sp); + RenderFrame(&frame_desc, "#%n %p %F %L\n", 0, frame->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); frame->ClearAll(); @@ -384,7 +390,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, } void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal) { + bool is_store, bool fatal, uptr *registers_frame) { ScopedReport R(fatal); SavedStackAllocations current_stack_allocations( GetCurrentThread()->stack_allocations()); @@ -400,13 +406,21 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, Thread *t = GetCurrentThread(); + sptr offset = + __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); + CHECK(offset >= 0 && offset < static_cast<sptr>(access_size)); tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t *tag_ptr = + reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, mem_tag, t->unique_id()); + if (offset != 0) + Printf("Invalid access starting at offset [%zu, %zu)\n", offset, + Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale))); Printf("%s", d.Default()); stack->Print(); @@ -417,7 +431,37 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, PrintTagsAroundAddr(tag_ptr); + if (registers_frame) + ReportRegisters(registers_frame, pc); + ReportErrorSummary(bug_type, stack); } +// See the frame breakdown defined in __hwasan_tag_mismatch (from +// hwasan_tag_mismatch_aarch64.S). +void ReportRegisters(uptr *frame, uptr pc) { + Printf("Registers where the failure occurred (pc %p):\n", pc); + + // We explicitly print a single line (4 registers/line) each iteration to + // reduce the amount of logcat error messages printed. Each Printf() will + // result in a new logcat line, irrespective of whether a newline is present, + // and so we wish to reduce the number of Printf() calls we have to make. + Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n", + frame[0], frame[1], frame[2], frame[3]); + Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", + frame[4], frame[5], frame[6], frame[7]); + Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n", + frame[8], frame[9], frame[10], frame[11]); + Printf(" x12 %016llx x13 %016llx x14 %016llx x15 %016llx\n", + frame[12], frame[13], frame[14], frame[15]); + Printf(" x16 %016llx x17 %016llx x18 %016llx x19 %016llx\n", + frame[16], frame[17], frame[18], frame[19]); + Printf(" x20 %016llx x21 %016llx x22 %016llx x23 %016llx\n", + frame[20], frame[21], frame[22], frame[23]); + Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", + frame[24], frame[25], frame[26], frame[27]); + Printf(" x28 %016llx x29 %016llx x30 %016llx\n", + frame[28], frame[29], frame[30]); +} + } // namespace __hwasan diff --git a/lib/hwasan/hwasan_report.h b/lib/hwasan/hwasan_report.h index 10fb20cc5..f03eb7a69 100644 --- a/lib/hwasan/hwasan_report.h +++ b/lib/hwasan/hwasan_report.h @@ -1,9 +1,8 @@ //===-- hwasan_report.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -23,11 +22,11 @@ namespace __hwasan { void ReportStats(); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store, bool fatal); + bool is_store, bool fatal, uptr *registers_frame); void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, uptr tail_size, const u8 *expected); - +void ReportRegisters(uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/lib/hwasan/hwasan_tag_mismatch_aarch64.S new file mode 100644 index 000000000..92f627480 --- /dev/null +++ b/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -0,0 +1,106 @@ +#include "sanitizer_common/sanitizer_asm.h" + +// The content of this file is AArch64-only: +#if defined(__aarch64__) + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x0, x1, x29, and x30. This function will +// "consume" these saved values and treats it as part of its own stack frame. +// In this sense, the __hwasan_check_* callee and this function "share" a stack +// frame. This allows us to omit having unwinding information (.cfi_*) present +// in every __hwasan_check_* function, therefore reducing binary size. This is +// particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// Frame from __hwasan_check_: +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | Unused 8-bytes for maintaining | +// | 16-byte SP alignment. | +// +---------------------------------+ +// | Return address (x30) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ +// | Frame address (x29) for caller | +// | of __hwasan_check_* | +// +---------------------------------+ <-- [SP + 232] +// | ... | +// | | +// | Stack frame space for x2 - x28. | +// | | +// | ... | +// +---------------------------------+ <-- [SP + 16] +// | | +// | Saved x1, as __hwasan_check_* | +// | clobbers it. | +// +---------------------------------+ +// | Saved x0, likewise above. | +// +---------------------------------+ <-- [x30 / SP] + +// This function takes two arguments: +// * x0: The address of read/write instruction that caused HWASan check fail. +// * x1: The tag size. + +.section .text +.file "hwasan_tag_mismatch_aarch64.S" +.global __hwasan_tag_mismatch +.type __hwasan_tag_mismatch, %function +__hwasan_tag_mismatch: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + add x29, sp, #232 + CFI_DEF_CFA(w29, 24) + CFI_OFFSET(w30, -16) + CFI_OFFSET(w29, -24) + + // Save the rest of the registers into the preallocated space left by + // __hwasan_check. + str x28, [sp, #224] + stp x26, x27, [sp, #208] + stp x24, x25, [sp, #192] + stp x22, x23, [sp, #176] + stp x20, x21, [sp, #160] + stp x18, x19, [sp, #144] + stp x16, x17, [sp, #128] + stp x14, x15, [sp, #112] + stp x12, x13, [sp, #96] + stp x10, x11, [sp, #80] + stp x8, x9, [sp, #64] + stp x6, x7, [sp, #48] + stp x4, x5, [sp, #32] + stp x2, x3, [sp, #16] + + // Pass the address of the frame to __hwasan_tag_mismatch_stub, so that it can + // extract the saved registers from this frame without having to worry about + // finding this frame. + mov x2, sp + + bl __hwasan_tag_mismatch_stub + CFI_ENDPROC + +.Lfunc_end0: + .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch + +#endif // defined(__aarch64__) + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cpp index 631c2813e..46dcddd42 100644 --- a/lib/hwasan/hwasan_thread.cc +++ b/lib/hwasan/hwasan_thread.cpp @@ -25,10 +25,13 @@ static u32 RandomSeed() { return seed; } +void Thread::InitRandomState() { + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; +} + void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { static u64 unique_id; unique_id_ = unique_id++; - random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h index 4830473f4..6fa592bfa 100644 --- a/lib/hwasan/hwasan_thread.h +++ b/lib/hwasan/hwasan_thread.h @@ -1,9 +1,8 @@ //===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + void InitRandomState(); void Destroy(); uptr stack_top() { return stack_top_; } @@ -67,11 +67,14 @@ class Thread { Print("Thread: "); } + uptr &vfork_spill() { return vfork_spill_; } + private: // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. void ClearShadowForThreadStackAndTLS(); void Print(const char *prefix); + uptr vfork_spill_; uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; diff --git a/lib/hwasan/hwasan_thread_list.cc b/lib/hwasan/hwasan_thread_list.cpp index a31eee84e..a31eee84e 100644 --- a/lib/hwasan/hwasan_thread_list.cc +++ b/lib/hwasan/hwasan_thread_list.cpp diff --git a/lib/hwasan/hwasan_thread_list.h b/lib/hwasan/hwasan_thread_list.h index 53747b51f..914b632d9 100644 --- a/lib/hwasan/hwasan_thread_list.h +++ b/lib/hwasan/hwasan_thread_list.h @@ -1,9 +1,8 @@ //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -109,38 +108,52 @@ struct ThreadStats { class HwasanThreadList { public: HwasanThreadList(uptr storage, uptr size) - : free_space_(storage), - free_space_end_(storage + size), - ring_buffer_size_(RingBufferSize()) {} + : free_space_(storage), free_space_end_(storage + size) { + // [storage, storage + size) is used as a vector of + // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements. + // Each element contains + // * a ring buffer at offset 0, + // * a Thread object at offset ring_buffer_size_. + ring_buffer_size_ = RingBufferSize(); + thread_alloc_size_ = + RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); + } Thread *CreateCurrentThread() { Thread *t; { SpinMutexLock l(&list_mutex_); t = free_list_.Pop(); - if (t) - internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); - else + if (t) { + uptr start = (uptr)t - ring_buffer_size_; + internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); + } else { t = AllocThread(); + } live_list_.Push(t); } - t->Init((uptr)(t + 1), ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); AddThreadStats(t); return t; } + void DontNeedThread(Thread *t) { + uptr start = (uptr)t - ring_buffer_size_; + ReleaseMemoryPagesToOS(start, start + thread_alloc_size_); + } + void ReleaseThread(Thread *t) { - // FIXME: madvise away the ring buffer? RemoveThreadStats(t); t->Destroy(); SpinMutexLock l(&list_mutex_); live_list_.Remove(t); free_list_.Push(t); + DontNeedThread(t); } Thread *GetThreadByBufferAddress(uptr p) { - uptr align = ring_buffer_size_ * 2; - return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) + + ring_buffer_size_); } uptr MemoryUsedPerThread() { @@ -176,15 +189,17 @@ class HwasanThreadList { private: Thread *AllocThread() { uptr align = ring_buffer_size_ * 2; - uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); - free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(IsAligned(free_space_, align)); + Thread *t = (Thread *)(free_space_ + ring_buffer_size_); + free_space_ += thread_alloc_size_; CHECK(free_space_ <= free_space_end_ && "out of thread memory"); - return (Thread *)(ring_buffer_start - sizeof(Thread)); + return t; } uptr free_space_; uptr free_space_end_; uptr ring_buffer_size_; + uptr thread_alloc_size_; ThreadListHead free_list_; ThreadListHead live_list_; diff --git a/lib/interception/CMakeLists.txt b/lib/interception/CMakeLists.txt index c0ac974d7..7f0de8189 100644 --- a/lib/interception/CMakeLists.txt +++ b/lib/interception/CMakeLists.txt @@ -4,13 +4,15 @@ set(INTERCEPTION_SOURCES interception_linux.cc interception_mac.cc interception_win.cc - interception_type_test.cc) + interception_type_test.cc + ) set(INTERCEPTION_HEADERS interception.h interception_linux.h interception_mac.h - interception_win.h) + interception_win.h + ) include_directories(..) diff --git a/lib/interception/interception.h b/lib/interception/interception.h index 87b2365fd..dacfa5ede 100644 --- a/lib/interception/interception.h +++ b/lib/interception/interception.h @@ -1,9 +1,8 @@ //===-- interception.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,11 +185,17 @@ const interpose_substitution substitution_##func_name[] \ #endif // SANITIZER_MAC #if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS -#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ extern "C" ret_type WRAP(func)(__VA_ARGS__); +// Declare an interceptor and its wrapper defined in a different translation +// unit (ex. asm). +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + extern "C" ret_type func(__VA_ARGS__); #else -#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) #endif // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR diff --git a/lib/interception/interception_linux.cc b/lib/interception/interception_linux.cc index 26bfcd8f6..d07f060b5 100644 --- a/lib/interception/interception_linux.cc +++ b/lib/interception/interception_linux.cc @@ -1,9 +1,8 @@ //===-- interception_linux.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,33 +18,57 @@ #include <dlfcn.h> // for dlsym() and dlvsym() +namespace __interception { + #if SANITIZER_NETBSD -#include "sanitizer_common/sanitizer_libc.h" +static int StrCmp(const char *s1, const char *s2) { + while (true) { + if (*s1 != *s2) + return false; + if (*s1 == 0) + return true; + s1++; + s2++; + } +} #endif -namespace __interception { -bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, - uptr real, uptr wrapper) { +static void *GetFuncAddr(const char *name) { #if SANITIZER_NETBSD - // XXX: Find a better way to handle renames - if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14"; + // FIXME: Find a better way to handle renames + if (StrCmp(name, "sigaction")) + name = "__sigaction14"; #endif - *func_addr = (uptr)dlsym(RTLD_NEXT, func_name); - if (!*func_addr) { + void *addr = dlsym(RTLD_NEXT, name); + if (!addr) { // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is // later in the library search order than the DSO that we are trying to // intercept, which means that we cannot intercept this function. We still // want the address of the real definition, though, so look it up using // RTLD_DEFAULT. - *func_addr = (uptr)dlsym(RTLD_DEFAULT, func_name); + addr = dlsym(RTLD_DEFAULT, name); } - return real == wrapper; + return addr; +} + +bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, + uptr wrapper) { + void *addr = GetFuncAddr(name); + *ptr_to_real = (uptr)addr; + return addr && (func == wrapper); } // Android and Solaris do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD -void *GetFuncAddrVer(const char *func_name, const char *ver) { - return dlvsym(RTLD_NEXT, func_name, ver); +static void *GetFuncAddr(const char *name, const char *ver) { + return dlvsym(RTLD_NEXT, name, ver); +} + +bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, + uptr func, uptr wrapper) { + void *addr = GetFuncAddr(name, ver); + *ptr_to_real = (uptr)addr; + return addr && (func == wrapper); } #endif // !SANITIZER_ANDROID diff --git a/lib/interception/interception_linux.h b/lib/interception/interception_linux.h index 765a186e5..e578da0cf 100644 --- a/lib/interception/interception_linux.h +++ b/lib/interception/interception_linux.h @@ -1,9 +1,8 @@ //===-- interception_linux.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,23 +22,27 @@ #define INTERCEPTION_LINUX_H namespace __interception { -// returns true if a function with the given name was found. -bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, - uptr real, uptr wrapper); -void *GetFuncAddrVer(const char *func_name, const char *ver); +bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, + uptr wrapper); +bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, + uptr func, uptr wrapper); } // namespace __interception -#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ - ::__interception::GetRealFunctionAddress( \ - #func, (::__interception::uptr *)&__interception::PTR_TO_REAL(func), \ - (::__interception::uptr) & (func), \ +#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ + ::__interception::InterceptFunction( \ + #func, \ + (::__interception::uptr *) & REAL(func), \ + (::__interception::uptr) & (func), \ (::__interception::uptr) & WRAP(func)) // Android, Solaris and OpenBSD do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - (::__interception::real_##func = (func##_type)( \ - unsigned long)::__interception::GetFuncAddrVer(#func, symver)) + ::__interception::InterceptFunction( \ + #func, symver, \ + (::__interception::uptr *) & REAL(func), \ + (::__interception::uptr) & (func), \ + (::__interception::uptr) & WRAP(func)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) diff --git a/lib/interception/interception_mac.cc b/lib/interception/interception_mac.cc index ea8072f8d..5bfc1514d 100644 --- a/lib/interception/interception_mac.cc +++ b/lib/interception/interception_mac.cc @@ -1,9 +1,8 @@ //===-- interception_mac.cc -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_mac.h b/lib/interception/interception_mac.h index 6f31fed47..eddedb895 100644 --- a/lib/interception/interception_mac.h +++ b/lib/interception/interception_mac.h @@ -1,9 +1,8 @@ //===-- interception_mac.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_type_test.cc b/lib/interception/interception_type_test.cc index 2b3a6d509..c00294a9b 100644 --- a/lib/interception/interception_type_test.cc +++ b/lib/interception/interception_type_test.cc @@ -1,9 +1,8 @@ //===-- interception_type_test.cc -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_win.cc b/lib/interception/interception_win.cc index cd13827e5..40bde0080 100644 --- a/lib/interception/interception_win.cc +++ b/lib/interception/interception_win.cc @@ -1,9 +1,8 @@ //===-- interception_linux.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -513,10 +512,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xc0854d: // 4d 85 c0 : test r8, r8 case 0xc2b60f: // 0f b6 c2 : movzx eax, dl case 0xc03345: // 45 33 c0 : xor r8d, r8d + case 0xc93345: // 45 33 c9 : xor r9d, r9d case 0xdb3345: // 45 33 DB : xor r11d, r11d case 0xd98b4c: // 4c 8b d9 : mov r11, rcx case 0xd28b4c: // 4c 8b d2 : mov r10, rdx case 0xc98b4c: // 4C 8B C9 : mov r9, rcx + case 0xc18b4c: // 4C 8B C1 : mov r8, rcx case 0xd2b60f: // 0f b6 d2 : movzx edx, dl case 0xca2b48: // 48 2b ca : sub rcx, rdx case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax] @@ -524,6 +525,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xd18b48: // 48 8b d1 : mov rdx, rcx case 0xdc8b4c: // 4c 8b dc : mov r11, rsp case 0xd18b4c: // 4c 8b d1 : mov r10, rcx + case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0 return 3; case 0xec8348: // 48 83 ec XX : sub rsp, XX @@ -555,6 +557,9 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx + case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx + case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9 + case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8 return 5; case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY return 6; diff --git a/lib/interception/interception_win.h b/lib/interception/interception_win.h index 71a44428f..459001301 100644 --- a/lib/interception/interception_win.h +++ b/lib/interception/interception_win.h @@ -1,9 +1,8 @@ //===-- interception_linux.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/tests/CMakeLists.txt b/lib/interception/tests/CMakeLists.txt index 1da0a455b..96bdda73e 100644 --- a/lib/interception/tests/CMakeLists.txt +++ b/lib/interception/tests/CMakeLists.txt @@ -21,6 +21,9 @@ set(INTERCEPTION_TEST_CFLAGS_COMMON -Werror=sign-compare -Wno-non-virtual-dtor) +set(INTERCEPTION_TEST_LINK_FLAGS_COMMON + ${COMPILER_RT_UNITTEST_LINK_FLAGS}) + # -gline-tables-only must be enough for these tests, so use it if possible. if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang") list(APPEND INTERCEPTION_TEST_CFLAGS_COMMON -gline-tables-only) diff --git a/lib/interception/tests/interception_linux_test.cc b/lib/interception/tests/interception_linux_test.cc index cc09aa09d..3e859cb1a 100644 --- a/lib/interception/tests/interception_linux_test.cc +++ b/lib/interception/tests/interception_linux_test.cc @@ -1,9 +1,8 @@ //===-- interception_linux_test.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,19 +33,19 @@ INTERCEPTOR(int, isdigit, int d) { namespace __interception { -TEST(Interception, GetRealFunctionAddress) { +TEST(Interception, InterceptFunction) { uptr malloc_address = 0; - EXPECT_TRUE(GetRealFunctionAddress("malloc", &malloc_address, 0, 0)); + EXPECT_TRUE(InterceptFunction("malloc", &malloc_address, 0, 0)); EXPECT_NE(0U, malloc_address); + EXPECT_FALSE(InterceptFunction("malloc", &malloc_address, 0, 1)); uptr dummy_address = 0; - EXPECT_TRUE( - GetRealFunctionAddress("dummy_doesnt_exist__", &dummy_address, 0, 0)); + EXPECT_FALSE(InterceptFunction("dummy_doesnt_exist__", &dummy_address, 0, 0)); EXPECT_EQ(0U, dummy_address); } TEST(Interception, Basic) { - ASSERT_TRUE(INTERCEPT_FUNCTION(isdigit)); + EXPECT_TRUE(INTERCEPT_FUNCTION(isdigit)); // After interception, the counter should be incremented. InterceptorFunctionCalled = 0; diff --git a/lib/interception/tests/interception_test_main.cc b/lib/interception/tests/interception_test_main.cc index 311da51ec..97e5b377a 100644 --- a/lib/interception/tests/interception_test_main.cc +++ b/lib/interception/tests/interception_test_main.cc @@ -1,9 +1,8 @@ //===-- interception_test_main.cc------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/tests/interception_win_test.cc b/lib/interception/tests/interception_win_test.cc index 37ef994f8..c3affc45b 100644 --- a/lib/interception/tests/interception_win_test.cc +++ b/lib/interception/tests/interception_win_test.cc @@ -1,9 +1,8 @@ //===-- interception_win_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -209,6 +208,24 @@ const u8 kUnpatchableCode6[] = { 0x90, 0x90, 0x90, 0x90, }; +const u8 kPatchableCode6[] = { + 0x48, 0x89, 0x54, 0x24, 0xBB, // mov QWORD PTR [rsp + 0xBB], rdx + 0x33, 0xC9, // xor ecx,ecx + 0xC3, // ret +}; + +const u8 kPatchableCode7[] = { + 0x4c, 0x89, 0x4c, 0x24, 0xBB, // mov QWORD PTR [rsp + 0xBB], r9 + 0x33, 0xC9, // xor ecx,ecx + 0xC3, // ret +}; + +const u8 kPatchableCode8[] = { + 0x4c, 0x89, 0x44, 0x24, 0xBB, // mov QWORD PTR [rsp + 0xBB], r8 + 0x33, 0xC9, // xor ecx,ecx + 0xC3, // ret +}; + // A buffer holding the dynamically generated code under test. u8* ActiveCode; const size_t ActiveCodeLength = 4096; @@ -508,7 +525,6 @@ TEST(Interception, PatchableFunction) { #endif EXPECT_TRUE(TestFunctionPatching(kPatchableCode4, override)); EXPECT_TRUE(TestFunctionPatching(kPatchableCode5, override)); - #if SANITIZER_WINDOWS64 EXPECT_TRUE(TestFunctionPatching(kLoadGlobalCode, override)); #endif @@ -573,7 +589,11 @@ TEST(Interception, PatchableFunctionWithHotPatch) { EXPECT_FALSE(TestFunctionPatching(kPatchableCode2, override, prefix)); EXPECT_FALSE(TestFunctionPatching(kPatchableCode3, override, prefix)); EXPECT_FALSE(TestFunctionPatching(kPatchableCode4, override, prefix)); - +#if SANITIZER_WINDOWS64 + EXPECT_TRUE(TestFunctionPatching(kPatchableCode6, override, prefix)); + EXPECT_TRUE(TestFunctionPatching(kPatchableCode7, override, prefix)); + EXPECT_TRUE(TestFunctionPatching(kPatchableCode8, override, prefix)); +#endif EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode1, override, prefix)); EXPECT_TRUE(TestFunctionPatching(kUnpatchableCode2, override, prefix)); EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode3, override, prefix)); diff --git a/lib/lsan/lsan.cc b/lib/lsan/lsan.cc index 93bced045..1074affcc 100644 --- a/lib/lsan/lsan.cc +++ b/lib/lsan/lsan.cc @@ -1,9 +1,8 @@ //=-- lsan.cc -------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,6 +32,24 @@ bool WordIsPoisoned(uptr addr) { } // namespace __lsan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __lsan; + uptr stack_top = 0, stack_bottom = 0; + ThreadContext *t; + if (StackTrace::WillUseFastUnwind(request_fast) && + (t = CurrentThreadContext())) { + stack_top = t->stack_end(); + stack_bottom = t->stack_begin(); + } + if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, stack_top, stack_bottom, true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); + } +} + using namespace __lsan; // NOLINT static void InitializeFlags() { @@ -72,7 +89,7 @@ static void InitializeFlags() { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + stack->Unwind(sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/lsan/lsan.h b/lib/lsan/lsan.h index 6baee81d0..9904ada4b 100644 --- a/lib/lsan/lsan.h +++ b/lib/lsan/lsan.h @@ -1,9 +1,8 @@ //=-- lsan.h --------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ #define GET_STACK_TRACE(max_size, fast) \ __sanitizer::BufferedStackTrace stack; \ - GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), nullptr, fast); + stack.Unwind(StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast, max_size); #define GET_STACK_TRACE_FATAL \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) @@ -41,24 +40,6 @@ void ReplaceSystemMalloc(); __lsan_init(); \ } while (0) -// Get the stack trace with the given pc and bp. -// The pc will be in the position 0 of the resulting stack trace. -// The bp may refer to the current frame or to the caller's frame. -ALWAYS_INLINE -void GetStackTrace(__sanitizer::BufferedStackTrace *stack, - __sanitizer::uptr max_depth, __sanitizer::uptr pc, - __sanitizer::uptr bp, void *context, bool fast) { - uptr stack_top = 0, stack_bottom = 0; - ThreadContext *t; - if (fast && (t = CurrentThreadContext())) { - stack_top = t->stack_end(); - stack_bottom = t->stack_begin(); - } - if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { - stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); - } -} - } // namespace __lsan extern bool lsan_inited; diff --git a/lib/lsan/lsan_allocator.cc b/lib/lsan/lsan_allocator.cc index 1b338bd59..8b13e4c02 100644 --- a/lib/lsan/lsan_allocator.cc +++ b/lib/lsan/lsan_allocator.cc @@ -1,9 +1,8 @@ //=-- lsan_allocator.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,6 +185,17 @@ void *lsan_realloc(void *p, uptr size, const StackTrace &stack) { return SetErrnoOnNull(Reallocate(stack, p, size, 1)); } +void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size, + const StackTrace &stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, &stack); + } + return lsan_realloc(ptr, nmemb * size, stack); +} + void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { return SetErrnoOnNull(Calloc(nmemb, size, stack)); } diff --git a/lib/lsan/lsan_allocator.h b/lib/lsan/lsan_allocator.h index 4c4e02fc0..e13970997 100644 --- a/lib/lsan/lsan_allocator.h +++ b/lib/lsan/lsan_allocator.h @@ -1,9 +1,8 @@ //=-- lsan_allocator.h ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,21 +51,14 @@ struct ChunkMetadata { #if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ defined(__arm__) -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -template <typename AddressSpaceView> -using ByteMapASVT = - TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; - template <typename AddressSpaceViewTy> struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __lsan::kRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = __lsan::ByteMapASVT<AddressSpaceView>; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -98,23 +90,11 @@ using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif template <typename AddressSpaceView> -using AllocatorCacheASVT = - SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; -using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; - -template <typename AddressSpaceView> -using SecondaryAllocatorASVT = - LargeMmapAllocator<NoOpMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, - AddressSpaceView>; - -template <typename AddressSpaceView> -using AllocatorASVT = - CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, - AllocatorCacheASVT<AddressSpaceView>, - SecondaryAllocatorASVT<AddressSpaceView>>; +using AllocatorASVT = CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; using Allocator = AllocatorASVT<LocalAddressSpaceView>; +using AllocatorCache = Allocator::AllocatorCache; -AllocatorCache *GetAllocatorCache(); +Allocator::AllocatorCache *GetAllocatorCache(); int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, const StackTrace &stack); @@ -123,6 +103,8 @@ void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack); void *lsan_malloc(uptr size, const StackTrace &stack); void lsan_free(void *p); void *lsan_realloc(void *p, uptr size, const StackTrace &stack); +void *lsan_reallocarray(void *p, uptr nmemb, uptr size, + const StackTrace &stack); void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack); void *lsan_valloc(uptr size, const StackTrace &stack); void *lsan_pvalloc(uptr size, const StackTrace &stack); diff --git a/lib/lsan/lsan_common.cc b/lib/lsan/lsan_common.cc index eaa5cadc8..7c842a152 100644 --- a/lib/lsan/lsan_common.cc +++ b/lib/lsan/lsan_common.cc @@ -1,9 +1,8 @@ //=-- lsan_common.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_common.h b/lib/lsan/lsan_common.h index 1d1e1e462..682e5f7d8 100644 --- a/lib/lsan/lsan_common.h +++ b/lib/lsan/lsan_common.h @@ -1,9 +1,8 @@ //=-- lsan_common.h -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_common_linux.cc b/lib/lsan/lsan_common_linux.cc index cdd7f032a..0e4abdce0 100644 --- a/lib/lsan/lsan_common_linux.cc +++ b/lib/lsan/lsan_common_linux.cc @@ -1,9 +1,8 @@ //=-- lsan_common_linux.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_common_mac.cc b/lib/lsan/lsan_common_mac.cc index a355cea96..14c2b3711 100644 --- a/lib/lsan/lsan_common_mac.cc +++ b/lib/lsan/lsan_common_mac.cc @@ -1,9 +1,8 @@ //=-- lsan_common_mac.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_flags.inc b/lib/lsan/lsan_flags.inc index e390e2ae5..9350f4bcd 100644 --- a/lib/lsan/lsan_flags.inc +++ b/lib/lsan/lsan_flags.inc @@ -1,9 +1,8 @@ //===-- lsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_interceptors.cc b/lib/lsan/lsan_interceptors.cc index a9bd2ba42..4a4c86a9d 100644 --- a/lib/lsan/lsan_interceptors.cc +++ b/lib/lsan/lsan_interceptors.cc @@ -1,9 +1,8 @@ //=-- lsan_interceptors.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -84,6 +83,12 @@ INTERCEPTOR(void*, realloc, void *q, uptr size) { return lsan_realloc(q, size, stack); } +INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) { + ENSURE_LSAN_INITED; + GET_STACK_TRACE_MALLOC; + return lsan_reallocarray(q, nmemb, size, stack); +} + INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) { ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; diff --git a/lib/lsan/lsan_linux.cc b/lib/lsan/lsan_linux.cc index c9749c745..c54f41ece 100644 --- a/lib/lsan/lsan_linux.cc +++ b/lib/lsan/lsan_linux.cc @@ -1,9 +1,8 @@ //=-- lsan_linux.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_mac.cc b/lib/lsan/lsan_mac.cc index 1a6f5f489..435f41b6f 100644 --- a/lib/lsan/lsan_mac.cc +++ b/lib/lsan/lsan_mac.cc @@ -1,9 +1,8 @@ //===-- lsan_mac.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_malloc_mac.cc b/lib/lsan/lsan_malloc_mac.cc index 94ffb6d02..34447b4b3 100644 --- a/lib/lsan/lsan_malloc_mac.cc +++ b/lib/lsan/lsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- lsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,6 +51,8 @@ using namespace __lsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __lsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/lsan/lsan_preinit.cc b/lib/lsan/lsan_preinit.cc index 5a190959c..5d0ad89a8 100644 --- a/lib/lsan/lsan_preinit.cc +++ b/lib/lsan/lsan_preinit.cc @@ -1,9 +1,8 @@ //===-- lsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_thread.cc b/lib/lsan/lsan_thread.cc index a25aff379..77f6a9236 100644 --- a/lib/lsan/lsan_thread.cc +++ b/lib/lsan/lsan_thread.cc @@ -1,9 +1,8 @@ //=-- lsan_thread.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -77,7 +76,7 @@ u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached) { /* arg */ nullptr); } -void ThreadStart(u32 tid, tid_t os_id, bool workerthread) { +void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type) { OnStartedArgs args; uptr stack_size = 0; uptr tls_size = 0; @@ -87,7 +86,7 @@ void ThreadStart(u32 tid, tid_t os_id, bool workerthread) { args.tls_end = args.tls_begin + tls_size; GetAllocatorCacheRange(&args.cache_begin, &args.cache_end); args.dtls = DTLS_Get(); - thread_registry->StartThread(tid, os_id, workerthread, &args); + thread_registry->StartThread(tid, os_id, thread_type, &args); } void ThreadFinish() { diff --git a/lib/lsan/lsan_thread.h b/lib/lsan/lsan_thread.h index b16d3d915..b869d066d 100644 --- a/lib/lsan/lsan_thread.h +++ b/lib/lsan/lsan_thread.h @@ -1,9 +1,8 @@ //=-- lsan_thread.h -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,7 +44,8 @@ class ThreadContext : public ThreadContextBase { void InitializeThreadRegistry(); -void ThreadStart(u32 tid, tid_t os_id, bool workerthread = false); +void ThreadStart(u32 tid, tid_t os_id, + ThreadType thread_type = ThreadType::Regular); void ThreadFinish(); u32 ThreadCreate(u32 tid, uptr uid, bool detached); void ThreadJoin(u32 tid); diff --git a/lib/msan/msan.cc b/lib/msan/msan.cc index ba2d5d593..c375afb77 100644 --- a/lib/msan/msan.cc +++ b/lib/msan/msan.cc @@ -1,9 +1,8 @@ //===-- msan.cc -----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -222,18 +221,6 @@ static void InitializeFlags() { if (f->store_context_size < 1) f->store_context_size = 1; } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - MsanThread *t = GetCurrentThread(); - if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - void PrintWarning(uptr pc, uptr bp) { PrintWarningWithOrigin(pc, bp, __msan_origin_tls); } @@ -314,6 +301,21 @@ u32 ChainOrigin(u32 id, StackTrace *stack) { } // namespace __msan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __msan; + MsanThread *t = GetCurrentThread(); + if (!t || !StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, false); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + // Interface. using namespace __msan; @@ -378,7 +380,7 @@ void __msan_warning_noreturn() { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + stack->Unwind(sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/msan/msan.h b/lib/msan/msan.h index 4c2e9f9e2..ac5f67e6a 100644 --- a/lib/msan/msan.h +++ b/lib/msan/msan.h @@ -1,9 +1,8 @@ //===-- msan.h --------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -289,6 +288,7 @@ void MsanDeallocate(StackTrace *stack, void *ptr); void *msan_malloc(uptr size, StackTrace *stack); void *msan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *msan_realloc(void *ptr, uptr size, StackTrace *stack); +void *msan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *msan_valloc(uptr size, StackTrace *stack); void *msan_pvalloc(uptr size, StackTrace *stack); void *msan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -313,9 +313,6 @@ struct SymbolizerScope { void PrintWarning(uptr pc, uptr bp); void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin); -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Unpoison first n function arguments. void UnpoisonParam(uptr n); void UnpoisonThreadLocalState(); @@ -329,23 +326,21 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ if (__msan_get_track_origins() && msan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) // For platforms which support slow unwinder only, we restrict the store context // size to 1, basically only storing the current pc. We do this because the slow // unwinder which is based on libunwind is not async signal safe and causes // random freezes in forking applications as well as in signal handlers. -#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() > 1 && msan_inited) { \ - if (!SANITIZER_CAN_FAST_UNWIND) \ - GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ - nullptr, false); \ - else \ - GetStackTrace(&stack, flags()->store_context_size, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_malloc); \ +#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() > 1 && msan_inited) { \ + int size = flags()->store_context_size; \ + if (!SANITIZER_CAN_FAST_UNWIND) \ + size = Min(size, 1); \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_malloc, size);\ } #define GET_STORE_STACK_TRACE \ @@ -354,8 +349,7 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ if (msan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/msan/msan_allocator.cc b/lib/msan/msan_allocator.cc index 053ab0280..181684001 100644 --- a/lib/msan/msan_allocator.cc +++ b/lib/msan/msan_allocator.cc @@ -1,9 +1,8 @@ //===-- msan_allocator.cc --------------------------- ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,81 +45,71 @@ struct MsanMapUnmapCallback { }; #if defined(__mips64) - static const uptr kMaxAllowedMallocSize = 2UL << 30; +static const uptr kMaxAllowedMallocSize = 2UL << 30; + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = 20; - static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; - typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - - struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __msan::ByteMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - }; - typedef SizeClassAllocator32<AP32> PrimaryAllocator; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; #elif defined(__x86_64__) #if SANITIZER_NETBSD || \ (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)) - static const uptr kAllocatorSpace = 0x700000000000ULL; +static const uptr kAllocatorSpace = 0x700000000000ULL; #else - static const uptr kAllocatorSpace = 0x600000000000ULL; +static const uptr kAllocatorSpace = 0x600000000000ULL; #endif - static const uptr kMaxAllowedMallocSize = 8UL << 30; - - struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = kAllocatorSpace; - static const uptr kSpaceSize = 0x40000000000; // 4T. - static const uptr kMetadataSize = sizeof(Metadata); - typedef DefaultSizeClassMap SizeClassMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - using AddressSpaceView = LocalAddressSpaceView; - }; +static const uptr kMaxAllowedMallocSize = 8UL << 30; + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = 0x40000000000; // 4T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; - typedef SizeClassAllocator64<AP64> PrimaryAllocator; +typedef SizeClassAllocator64<AP64> PrimaryAllocator; #elif defined(__powerpc64__) - static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G - - struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = 0x300000000000; - static const uptr kSpaceSize = 0x020000000000; // 2T. - static const uptr kMetadataSize = sizeof(Metadata); - typedef DefaultSizeClassMap SizeClassMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - using AddressSpaceView = LocalAddressSpaceView; - }; - - typedef SizeClassAllocator64<AP64> PrimaryAllocator; +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = 0x300000000000; + static const uptr kSpaceSize = 0x020000000000; // 2T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; + +typedef SizeClassAllocator64<AP64> PrimaryAllocator; #elif defined(__aarch64__) - static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = 20; - static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; - typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - - struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __msan::ByteMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - }; - typedef SizeClassAllocator32<AP32> PrimaryAllocator; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MsanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; static Allocator allocator; static AllocatorCache fallback_allocator_cache; @@ -270,6 +259,16 @@ void *msan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(MsanReallocate(stack, ptr, size, sizeof(u64))); } +void *msan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return msan_realloc(ptr, nmemb * size, stack); +} + void *msan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull(MsanAllocate(stack, size, GetPageSizeCached(), false)); } diff --git a/lib/msan/msan_allocator.h b/lib/msan/msan_allocator.h index 407942e54..42a5022c9 100644 --- a/lib/msan/msan_allocator.h +++ b/lib/msan/msan_allocator.h @@ -1,9 +1,8 @@ //===-- msan_allocator.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_chained_origin_depot.cc b/lib/msan/msan_chained_origin_depot.cc index e2796fd46..6c634252e 100644 --- a/lib/msan/msan_chained_origin_depot.cc +++ b/lib/msan/msan_chained_origin_depot.cc @@ -1,9 +1,8 @@ //===-- msan_chained_origin_depot.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_chained_origin_depot.h b/lib/msan/msan_chained_origin_depot.h index f7a71cef2..2b4cb3647 100644 --- a/lib/msan/msan_chained_origin_depot.h +++ b/lib/msan/msan_chained_origin_depot.h @@ -1,9 +1,8 @@ //===-- msan_chained_origin_depot.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_flags.h b/lib/msan/msan_flags.h index 4fc6d172a..836dbbaa1 100644 --- a/lib/msan/msan_flags.h +++ b/lib/msan/msan_flags.h @@ -1,9 +1,8 @@ //===-- msan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_flags.inc b/lib/msan/msan_flags.inc index a7ff6c586..e6a26015a 100644 --- a/lib/msan/msan_flags.inc +++ b/lib/msan/msan_flags.inc @@ -1,9 +1,8 @@ //===-- msan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc index 497f943a8..af9d14029 100644 --- a/lib/msan/msan_interceptors.cc +++ b/lib/msan/msan_interceptors.cc @@ -1,9 +1,8 @@ //===-- msan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -908,6 +907,11 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { return msan_realloc(ptr, size, &stack); } +INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return msan_reallocarray(ptr, nmemb, size, &stack); +} + INTERCEPTOR(void *, malloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!msan_inited)) @@ -1119,8 +1123,12 @@ void MSanAtExitWrapper() { void MSanCxaAtExitWrapper(void *arg) { UnpoisonParam(1); MSanAtExitRecord *r = (MSanAtExitRecord *)arg; + // libc before 2.27 had race which caused occasional double handler execution + // https://sourceware.org/ml/libc-alpha/2017-08/msg01204.html + if (!r->func) + return; r->func(r->arg); - InternalFree(r); + r->func = nullptr; } static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso); @@ -1184,14 +1192,14 @@ INTERCEPTOR(int, fork, void) { // NetBSD ships with openpty(3) in -lutil, that needs to be prebuilt explicitly // with MSan. #if SANITIZER_LINUX -INTERCEPTOR(int, openpty, int *amaster, int *aslave, char *name, +INTERCEPTOR(int, openpty, int *aparent, int *aworker, char *name, const void *termp, const void *winp) { ENSURE_MSAN_INITED(); InterceptorScope interceptor_scope; - int res = REAL(openpty)(amaster, aslave, name, termp, winp); + int res = REAL(openpty)(aparent, aworker, name, termp, winp); if (!res) { - __msan_unpoison(amaster, sizeof(*amaster)); - __msan_unpoison(aslave, sizeof(*aslave)); + __msan_unpoison(aparent, sizeof(*aparent)); + __msan_unpoison(aworker, sizeof(*aworker)); } return res; } @@ -1203,13 +1211,13 @@ INTERCEPTOR(int, openpty, int *amaster, int *aslave, char *name, // NetBSD ships with forkpty(3) in -lutil, that needs to be prebuilt explicitly // with MSan. #if SANITIZER_LINUX -INTERCEPTOR(int, forkpty, int *amaster, char *name, const void *termp, +INTERCEPTOR(int, forkpty, int *aparent, char *name, const void *termp, const void *winp) { ENSURE_MSAN_INITED(); InterceptorScope interceptor_scope; - int res = REAL(forkpty)(amaster, name, termp, winp); + int res = REAL(forkpty)(aparent, name, termp, winp); if (res != -1) - __msan_unpoison(amaster, sizeof(*amaster)); + __msan_unpoison(aparent, sizeof(*aparent)); return res; } #define MSAN_MAYBE_INTERCEPT_FORKPTY INTERCEPT_FUNCTION(forkpty) @@ -1240,13 +1248,13 @@ int OnExit() { #define MSAN_INTERCEPT_FUNC(name) \ do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ + if (!INTERCEPT_FUNCTION(name)) \ VReport(1, "MemorySanitizer: failed to intercept '" #name "'\n"); \ } while (0) #define MSAN_INTERCEPT_FUNC_VER(name, ver) \ do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ VReport( \ 1, "MemorySanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ } while (0) @@ -1594,6 +1602,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(malloc); INTERCEPT_FUNCTION(calloc); INTERCEPT_FUNCTION(realloc); + INTERCEPT_FUNCTION(reallocarray); INTERCEPT_FUNCTION(free); MSAN_MAYBE_INTERCEPT_CFREE; MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE; diff --git a/lib/msan/msan_interface_internal.h b/lib/msan/msan_interface_internal.h index 9a67cbc9b..09680f67a 100644 --- a/lib/msan/msan_interface_internal.h +++ b/lib/msan/msan_interface_internal.h @@ -1,9 +1,8 @@ //===-- msan_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_linux.cc b/lib/msan/msan_linux.cc index 0b0208884..3b6e6cb85 100644 --- a/lib/msan/msan_linux.cc +++ b/lib/msan/msan_linux.cc @@ -1,9 +1,8 @@ //===-- msan_linux.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_new_delete.cc b/lib/msan/msan_new_delete.cc index a0959aec5..750981eb5 100644 --- a/lib/msan/msan_new_delete.cc +++ b/lib/msan/msan_new_delete.cc @@ -1,9 +1,8 @@ //===-- msan_new_delete.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_origin.h b/lib/msan/msan_origin.h index 36c168b85..26a4e7eb9 100644 --- a/lib/msan/msan_origin.h +++ b/lib/msan/msan_origin.h @@ -1,9 +1,8 @@ //===-- msan_origin.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_poisoning.cc b/lib/msan/msan_poisoning.cc index 7420d9469..5ea01f51a 100644 --- a/lib/msan/msan_poisoning.cc +++ b/lib/msan/msan_poisoning.cc @@ -1,9 +1,8 @@ //===-- msan_poisoning.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_poisoning.h b/lib/msan/msan_poisoning.h index edacbeeab..270f1e249 100644 --- a/lib/msan/msan_poisoning.h +++ b/lib/msan/msan_poisoning.h @@ -1,9 +1,8 @@ //===-- msan_poisoning.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_report.cc b/lib/msan/msan_report.cc index 2f0cc8d37..73bce3972 100644 --- a/lib/msan/msan_report.cc +++ b/lib/msan/msan_report.cc @@ -1,9 +1,8 @@ //===-- msan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_report.h b/lib/msan/msan_report.h index 73840e417..0965b8cb6 100644 --- a/lib/msan/msan_report.h +++ b/lib/msan/msan_report.h @@ -1,9 +1,8 @@ //===-- msan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/msan/msan_thread.h b/lib/msan/msan_thread.h index ed22e67ed..808780cd5 100644 --- a/lib/msan/msan_thread.h +++ b/lib/msan/msan_thread.h @@ -1,9 +1,8 @@ //===-- msan_thread.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/tests/CMakeLists.txt b/lib/msan/tests/CMakeLists.txt index e9f4e34bf..eceb11cdd 100644 --- a/lib/msan/tests/CMakeLists.txt +++ b/lib/msan/tests/CMakeLists.txt @@ -47,11 +47,10 @@ set(MSAN_UNITTEST_INSTRUMENTED_CFLAGS -mllvm -msan-keep-going=1 ) set(MSAN_UNITTEST_LINK_FLAGS + ${COMPILER_RT_UNITTEST_LINK_FLAGS} -fsanitize=memory - # Don't need -stdlib=libc++ because we explicitly list libc++.so in the linker + # Don't need -stdlib=libc++ because we explicitly list libc++.a in the linker # inputs. - # FIXME: we build libcxx without cxxabi and need libstdc++ to provide it. - -lstdc++ ) append_list_if(COMPILER_RT_HAS_LIBDL -ldl MSAN_UNITTEST_LINK_FLAGS) @@ -75,7 +74,7 @@ macro(msan_link_shared so_list so_name arch kind) endif() clang_link_shared(${output_so} OBJECTS ${SOURCE_OBJECTS} - LINK_FLAGS ${TARGET_LINK_FLAGS} ${SOURCE_LINK_FLAGS} + LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS} ${TARGET_LINK_FLAGS} ${SOURCE_LINK_FLAGS} DEPS ${SOURCE_DEPS}) list(APPEND ${so_list} ${output_so}) endmacro() @@ -116,16 +115,16 @@ macro(add_msan_tests_for_arch arch kind cflags) endif() get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) add_compiler_rt_test(MsanUnitTests "Msan-${arch}${kind}-Test" ${arch} - OBJECTS ${MSAN_TEST_OBJECTS} ${MSAN_LIBCXX_SO} + OBJECTS ${MSAN_TEST_OBJECTS} "${MSAN_LIBCXX_DIR}/libc++.a" DEPS ${MSAN_TEST_DEPS} LINK_FLAGS ${MSAN_UNITTEST_LINK_FLAGS} - ${TARGET_LINK_FLAGS} - "-Wl,-rpath=${CMAKE_CURRENT_BINARY_DIR}" - "-Wl,-rpath=${LIBCXX_PREFIX}/lib") + ${TARGET_LINK_FLAGS}) endmacro() # We should only build MSan unit tests if we can build instrumented libcxx. -if(COMPILER_RT_CAN_EXECUTE_TESTS AND COMPILER_RT_LIBCXX_PATH) +if(COMPILER_RT_CAN_EXECUTE_TESTS AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) foreach(arch ${MSAN_SUPPORTED_ARCH}) get_target_flags_for_arch(${arch} TARGET_CFLAGS) set(LIBCXX_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}) @@ -133,7 +132,7 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS AND COMPILER_RT_LIBCXX_PATH) DEPS ${MSAN_RUNTIME_LIBRARIES} CFLAGS ${MSAN_LIBCXX_CFLAGS} ${TARGET_CFLAGS} USE_TOOLCHAIN) - set(MSAN_LIBCXX_SO ${LIBCXX_PREFIX}/lib/libc++.so) + set(MSAN_LIBCXX_DIR ${LIBCXX_PREFIX}/lib/) add_msan_tests_for_arch(${arch} "" "") add_msan_tests_for_arch(${arch} "-with-call" diff --git a/lib/msan/tests/msan_loadable.cc b/lib/msan/tests/msan_loadable.cc index 06e880f90..b5bc7ff40 100644 --- a/lib/msan/tests/msan_loadable.cc +++ b/lib/msan/tests/msan_loadable.cc @@ -1,9 +1,8 @@ //===-- msan_loadable.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/tests/msan_test.cc b/lib/msan/tests/msan_test.cc index 19f46abdc..9d2f5a73b 100644 --- a/lib/msan/tests/msan_test.cc +++ b/lib/msan/tests/msan_test.cc @@ -1,9 +1,8 @@ //===-- msan_test.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -170,7 +169,7 @@ static bool TrackingOrigins() { #define EXPECT_POISONED(x) ExpectPoisoned(x) -template<typename T> +template <typename T> void ExpectPoisoned(const T& t) { EXPECT_NE(-1, __msan_test_shadow((void*)&t, sizeof(t))); } @@ -2124,6 +2123,16 @@ TEST(MemorySanitizer, wcrtomb) { EXPECT_EQ(buff[0], 'a'); } +TEST(MemorySanitizer, wctomb) { + wchar_t x = L'a'; + char buff[10]; + wctomb(nullptr, x); + int res = wctomb(buff, x); + EXPECT_EQ(res, 1); + EXPECT_EQ(buff[0], 'a'); + EXPECT_POISONED(buff[1]); +} + TEST(MemorySanitizer, wmemset) { wchar_t x[25]; break_optimization(x); @@ -2580,6 +2589,14 @@ TEST(MemorySanitizer, sigprocmask) { EXPECT_NOT_POISONED(s); } +TEST(MemorySanitizer, pthread_sigmask) { + sigset_t s; + EXPECT_POISONED(s); + int res = pthread_sigmask(SIG_BLOCK, 0, &s); + ASSERT_EQ(0, res); + EXPECT_NOT_POISONED(s); +} + struct StructWithDtor { ~StructWithDtor(); }; @@ -4637,3 +4654,147 @@ TEST(MemorySanitizer, MallocUsableSizeTest) { delete int_ptr; } #endif // SANITIZER_TEST_HAS_MALLOC_USABLE_SIZE + +#ifdef __x86_64__ +static bool HaveBmi() { + U4 a = 0, b = 0, c = 0, d = 0; + asm("cpuid\n\t" : "=a"(a), "=D"(b), "=c"(c), "=d"(d) : "a"(7)); + const U4 kBmi12Mask = (1U<<3) | (1U<<8); + return (b & kBmi12Mask) == kBmi12Mask; +} + +__attribute__((target("bmi,bmi2"))) +static void TestBZHI() { + EXPECT_NOT_POISONED( + __builtin_ia32_bzhi_si(Poisoned<U4>(0xABCDABCD, 0xFF000000), 24)); + EXPECT_POISONED( + __builtin_ia32_bzhi_si(Poisoned<U4>(0xABCDABCD, 0xFF800000), 24)); + // Second operand saturates. + EXPECT_POISONED( + __builtin_ia32_bzhi_si(Poisoned<U4>(0xABCDABCD, 0x80000000), 240)); + // Any poison in the second operand poisons output. + EXPECT_POISONED( + __builtin_ia32_bzhi_si(0xABCDABCD, Poisoned<U4>(1, 1))); + EXPECT_POISONED( + __builtin_ia32_bzhi_si(0xABCDABCD, Poisoned<U4>(1, 0x80000000))); + EXPECT_POISONED( + __builtin_ia32_bzhi_si(0xABCDABCD, Poisoned<U4>(1, 0xFFFFFFFF))); + + EXPECT_NOT_POISONED( + __builtin_ia32_bzhi_di(Poisoned<U8>(0xABCDABCDABCDABCD, 0xFF00000000000000ULL), 56)); + EXPECT_POISONED( + __builtin_ia32_bzhi_di(Poisoned<U8>(0xABCDABCDABCDABCD, 0xFF80000000000000ULL), 56)); + // Second operand saturates. + EXPECT_POISONED( + __builtin_ia32_bzhi_di(Poisoned<U8>(0xABCDABCDABCDABCD, 0x8000000000000000ULL), 240)); + // Any poison in the second operand poisons output. + EXPECT_POISONED( + __builtin_ia32_bzhi_di(0xABCDABCDABCDABCD, Poisoned<U8>(1, 1))); + EXPECT_POISONED( + __builtin_ia32_bzhi_di(0xABCDABCDABCDABCD, Poisoned<U8>(1, 0x8000000000000000ULL))); + EXPECT_POISONED( + __builtin_ia32_bzhi_di(0xABCDABCDABCDABCD, Poisoned<U8>(1, 0xFFFFFFFF00000000ULL))); +} + +inline U4 bextr_imm(U4 start, U4 len) { + start &= 0xFF; + len &= 0xFF; + return (len << 8) | start; +} + +__attribute__((target("bmi,bmi2"))) +static void TestBEXTR() { + EXPECT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(0, 8))); + EXPECT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(7, 8))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(8, 8))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(8, 800))); + EXPECT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(7, 800))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u32(Poisoned<U4>(0xABCDABCD, 0xFF), bextr_imm(5, 0))); + + EXPECT_POISONED( + __builtin_ia32_bextr_u32(0xABCDABCD, Poisoned<U4>(bextr_imm(7, 800), 1))); + EXPECT_POISONED(__builtin_ia32_bextr_u32( + 0xABCDABCD, Poisoned<U4>(bextr_imm(7, 800), 0x80000000))); + + EXPECT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(0, 8))); + EXPECT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(7, 8))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(8, 8))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(8, 800))); + EXPECT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(7, 800))); + EXPECT_NOT_POISONED( + __builtin_ia32_bextr_u64(Poisoned<U8>(0xABCDABCD, 0xFF), bextr_imm(5, 0))); + + // Poison in the top half. + EXPECT_NOT_POISONED(__builtin_ia32_bextr_u64( + Poisoned<U8>(0xABCDABCD, 0xFF0000000000), bextr_imm(32, 8))); + EXPECT_POISONED(__builtin_ia32_bextr_u64( + Poisoned<U8>(0xABCDABCD, 0xFF0000000000), bextr_imm(32, 9))); + + EXPECT_POISONED( + __builtin_ia32_bextr_u64(0xABCDABCD, Poisoned<U8>(bextr_imm(7, 800), 1))); + EXPECT_POISONED(__builtin_ia32_bextr_u64( + 0xABCDABCD, Poisoned<U8>(bextr_imm(7, 800), 0x80000000))); +} + +__attribute__((target("bmi,bmi2"))) +static void TestPDEP() { + U4 x = Poisoned<U4>(0, 0xFF00); + EXPECT_NOT_POISONED(__builtin_ia32_pdep_si(x, 0xFF)); + EXPECT_POISONED(__builtin_ia32_pdep_si(x, 0x1FF)); + EXPECT_NOT_POISONED(__builtin_ia32_pdep_si(x, 0xFF00)); + EXPECT_POISONED(__builtin_ia32_pdep_si(x, 0x1FF00)); + + EXPECT_NOT_POISONED(__builtin_ia32_pdep_si(x, 0x1FF00) & 0xFF); + EXPECT_POISONED(__builtin_ia32_pdep_si(0, Poisoned<U4>(0xF, 1))); + + U8 y = Poisoned<U8>(0, 0xFF00); + EXPECT_NOT_POISONED(__builtin_ia32_pdep_di(y, 0xFF)); + EXPECT_POISONED(__builtin_ia32_pdep_di(y, 0x1FF)); + EXPECT_NOT_POISONED(__builtin_ia32_pdep_di(y, 0xFF0000000000)); + EXPECT_POISONED(__builtin_ia32_pdep_di(y, 0x1FF000000000000)); + + EXPECT_NOT_POISONED(__builtin_ia32_pdep_di(y, 0x1FF00) & 0xFF); + EXPECT_POISONED(__builtin_ia32_pdep_di(0, Poisoned<U4>(0xF, 1))); +} + +__attribute__((target("bmi,bmi2"))) +static void TestPEXT() { + U4 x = Poisoned<U4>(0, 0xFF00); + EXPECT_NOT_POISONED(__builtin_ia32_pext_si(x, 0xFF)); + EXPECT_POISONED(__builtin_ia32_pext_si(x, 0x1FF)); + EXPECT_POISONED(__builtin_ia32_pext_si(x, 0x100)); + EXPECT_POISONED(__builtin_ia32_pext_si(x, 0x1000)); + EXPECT_NOT_POISONED(__builtin_ia32_pext_si(x, 0x10000)); + + EXPECT_POISONED(__builtin_ia32_pext_si(0xFF00, Poisoned<U4>(0xFF, 1))); + + U8 y = Poisoned<U8>(0, 0xFF0000000000); + EXPECT_NOT_POISONED(__builtin_ia32_pext_di(y, 0xFF00000000)); + EXPECT_POISONED(__builtin_ia32_pext_di(y, 0x1FF00000000)); + EXPECT_POISONED(__builtin_ia32_pext_di(y, 0x10000000000)); + EXPECT_POISONED(__builtin_ia32_pext_di(y, 0x100000000000)); + EXPECT_NOT_POISONED(__builtin_ia32_pext_di(y, 0x1000000000000)); + + EXPECT_POISONED(__builtin_ia32_pext_di(0xFF00, Poisoned<U8>(0xFF, 1))); +} + +TEST(MemorySanitizer, Bmi) { + if (HaveBmi()) { + TestBZHI(); + TestBEXTR(); + TestPDEP(); + TestPEXT(); + } +} +#endif // defined(__x86_64__) diff --git a/lib/msan/tests/msan_test_config.h b/lib/msan/tests/msan_test_config.h index 5404c434d..df906e30b 100644 --- a/lib/msan/tests/msan_test_config.h +++ b/lib/msan/tests/msan_test_config.h @@ -1,9 +1,8 @@ //===-- msan_test_config.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/tests/msan_test_main.cc b/lib/msan/tests/msan_test_main.cc index c8c5fefb1..d9839dc2b 100644 --- a/lib/msan/tests/msan_test_main.cc +++ b/lib/msan/tests/msan_test_main.cc @@ -1,9 +1,8 @@ //===-- msan_test_main.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/profile/CMakeLists.txt b/lib/profile/CMakeLists.txt index 488673dd2..9774be6a8 100644 --- a/lib/profile/CMakeLists.txt +++ b/lib/profile/CMakeLists.txt @@ -62,6 +62,7 @@ set(PROFILE_SOURCES InstrProfilingPlatformFuchsia.c InstrProfilingPlatformLinux.c InstrProfilingPlatformOther.c + InstrProfilingPlatformWindows.c InstrProfilingRuntime.cc InstrProfilingUtil.c) diff --git a/lib/profile/GCDAProfiling.c b/lib/profile/GCDAProfiling.c index 0665a680c..498c05900 100644 --- a/lib/profile/GCDAProfiling.c +++ b/lib/profile/GCDAProfiling.c @@ -1,9 +1,8 @@ /*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===*| |* @@ -269,14 +268,14 @@ static int map_file() { mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); if (mmap_handle == NULL) { - fprintf(stderr, "profiling: %s: cannot create file mapping: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n", + filename, GetLastError()); return -1; } write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); if (write_buffer == NULL) { - fprintf(stderr, "profiling: %s: cannot map: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename, GetLastError()); CloseHandle(mmap_handle); return -1; @@ -298,18 +297,18 @@ static int map_file() { static void unmap_file() { #if defined(_WIN32) if (!FlushViewOfFile(write_buffer, file_size)) { - fprintf(stderr, "profiling: %s: cannot flush mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename, GetLastError()); } if (!UnmapViewOfFile(write_buffer)) { - fprintf(stderr, "profiling: %s: cannot unmap mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, GetLastError()); } if (!CloseHandle(mmap_handle)) { - fprintf(stderr, "profiling: %s: cannot close file mapping handle: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n", + filename, GetLastError()); } mmap_handle = NULL; diff --git a/lib/profile/InstrProfData.inc b/lib/profile/InstrProfData.inc index 454620ed9..749781b9a 100644 --- a/lib/profile/InstrProfData.inc +++ b/lib/profile/InstrProfData.inc @@ -1,9 +1,8 @@ /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -170,7 +169,7 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) /* VALUE_PROF_KIND start */ #ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value) +#define VALUE_PROF_KIND(Enumerator, Value, Descr) #else #define INSTR_PROF_DATA_DEFINED #endif @@ -183,16 +182,16 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) * For this remapping the ProfData is used. ProfData contains both the function * name hash and the function address. */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1) +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -250,22 +249,25 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,") + INSTR_PROF_DATA_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,") + INSTR_PROF_CNTS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,") + INSTR_PROF_NAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,") + INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,") + INSTR_PROF_VNODES_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,") + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -636,10 +638,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -655,13 +659,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap -/* Win32 */ -#define INSTR_PROF_DATA_COFF .lprfd -#define INSTR_PROF_NAME_COFF .lprfn -#define INSTR_PROF_CNTS_COFF .lprfc -#define INSTR_PROF_VALS_COFF .lprfv -#define INSTR_PROF_VNODES_COFF .lprfnd -#define INSTR_PROF_COVMAP_COFF .lcovmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -675,32 +683,30 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) /* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) #endif -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) -#define INSTR_PROF_COVMAP_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) -#define INSTR_PROF_VALS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) -#define INSTR_PROF_VNODES_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -735,6 +741,12 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ +#ifndef INSTR_ORDER_FILE_INC +// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ #else #undef INSTR_PROF_DATA_DEFINED #endif diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c index 00b31e1ee..299cf3177 100644 --- a/lib/profile/InstrProfiling.c +++ b/lib/profile/InstrProfiling.c @@ -1,9 +1,8 @@ /*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h index 3ef7eacf0..63aebaf79 100644 --- a/lib/profile/InstrProfiling.h +++ b/lib/profile/InstrProfiling.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -16,7 +15,7 @@ #include "InstrProfData.inc" enum ValueKind { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, #include "InstrProfData.inc" }; @@ -65,6 +64,7 @@ uint64_t *__llvm_profile_begin_counters(void); uint64_t *__llvm_profile_end_counters(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +uint32_t *__llvm_profile_begin_orderfile(); /*! * \brief Clear profile counters to zero. @@ -121,6 +121,7 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, */ int __llvm_profile_write_file(void); +int __llvm_orderfile_write_file(void); /*! * \brief this is a wrapper interface to \c __llvm_profile_write_file. * After this interface is invoked, a arleady dumped flag will be set @@ -143,6 +144,8 @@ int __llvm_profile_write_file(void); */ int __llvm_profile_dump(void); +int __llvm_orderfile_dump(void); + /*! * \brief Set the filename for writing instrumentation data. * diff --git a/lib/profile/InstrProfilingBuffer.c b/lib/profile/InstrProfilingBuffer.c index a7e852f53..5bdeb8e32 100644 --- a/lib/profile/InstrProfilingBuffer.c +++ b/lib/profile/InstrProfilingBuffer.c @@ -1,9 +1,8 @@ /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c index c4cf3ccd7..8c32bde9d 100644 --- a/lib/profile/InstrProfilingFile.c +++ b/lib/profile/InstrProfilingFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -112,6 +111,15 @@ static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return 0; } +/* TODO: make buffer size controllable by an internal option, and compiler can pass the size + to runtime via a variable. */ +static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { + if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != + INSTR_ORDER_FILE_BUFFER_SIZE) + return 1; + return 0; +} + static void initFileWriter(ProfDataWriter *This, FILE *File) { This->Write = fileWriter; This->WriterCtx = File; @@ -261,6 +269,27 @@ static int writeFile(const char *OutputName) { return RetVal; } +/* Write order data to file \c OutputName. */ +static int writeOrderFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + OutputFile = fopen(OutputName, "w"); + + if (!OutputFile) { + PROF_WARN("can't open file with mode ab: %s\n", OutputName); + return -1; + } + + FreeHook = &free; + setupIOBuffer(); + const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); + RetVal = orderFileWriter(OutputFile, DataBegin); + + fclose(OutputFile); + return RetVal; +} + static void truncateCurrentFile(void) { const char *Filename; char *FilenameBuf; @@ -649,6 +678,62 @@ int __llvm_profile_dump(void) { return rc; } +/* Order file data will be saved in a file with suffx .order. */ +static const char *OrderFileSuffix = ".order"; + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_write_file(void) { + int rc, Length, LengthBeforeAppend, SuffixLength; + const char *Filename; + char *FilenameBuf; + int PDeathSig = 0; + + SuffixLength = strlen(OrderFileSuffix); + Length = getCurFilenameLength() + SuffixLength; + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 1); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + return -1; + } + + /* Append order file suffix */ + LengthBeforeAppend = strlen(Filename); + memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); + FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + // Temporarily suspend getting SIGKILL when the parent exits. + PDeathSig = lprofSuspendSigKill(); + + /* Write order data to the file. */ + rc = writeOrderFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_dump(void) { + int rc = __llvm_orderfile_write_file(); + return rc; +} + static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } COMPILER_RT_VISIBILITY diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h index 40540ab5f..66f8a0677 100644 --- a/lib/profile/InstrProfilingInternal.h +++ b/lib/profile/InstrProfilingInternal.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingMerge.c b/lib/profile/InstrProfilingMerge.c index 2397250fb..44dce7cc9 100644 --- a/lib/profile/InstrProfilingMerge.c +++ b/lib/profile/InstrProfilingMerge.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===* |* This file defines the API needed for in-process merging of profile data diff --git a/lib/profile/InstrProfilingMergeFile.c b/lib/profile/InstrProfilingMergeFile.c index dc1bc9762..b853f15b4 100644 --- a/lib/profile/InstrProfilingMergeFile.c +++ b/lib/profile/InstrProfilingMergeFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------=== |* This file defines APIs needed to support in-process merging for profile data @@ -21,6 +20,7 @@ /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ +COMPILER_RT_VISIBILITY void lprofMergeValueProfData(ValueProfData *SrcValueProfData, __llvm_profile_data *DstData) { unsigned I, S, V, DstIndex = 0; diff --git a/lib/profile/InstrProfilingNameVar.c b/lib/profile/InstrProfilingNameVar.c index 264568fbc..2d67a55b9 100644 --- a/lib/profile/InstrProfilingNameVar.c +++ b/lib/profile/InstrProfilingNameVar.c @@ -1,9 +1,8 @@ /*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingPlatformDarwin.c b/lib/profile/InstrProfilingPlatformDarwin.c index 8931abadd..23bdb7f37 100644 --- a/lib/profile/InstrProfilingPlatformDarwin.c +++ b/lib/profile/InstrProfilingPlatformDarwin.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -13,28 +12,31 @@ /* Use linker magic to find the bounds of the Data section. */ COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern char - NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY -extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint32_t + OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { @@ -50,6 +52,8 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformFuchsia.c b/lib/profile/InstrProfilingPlatformFuchsia.c index a50602ded..80beb4145 100644 --- a/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/lib/profile/InstrProfilingPlatformFuchsia.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -71,7 +70,7 @@ static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return -1; /* Create VMO to hold the profile data. */ - Status = _zx_vmo_create(0, 0, &__llvm_profile_vmo); + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &__llvm_profile_vmo); if (Status != ZX_OK) return -1; diff --git a/lib/profile/InstrProfilingPlatformLinux.c b/lib/profile/InstrProfilingPlatformLinux.c index 3764df1d8..becfe1fd9 100644 --- a/lib/profile/InstrProfilingPlatformLinux.c +++ b/lib/profile/InstrProfilingPlatformLinux.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -14,14 +13,15 @@ #include "InstrProfiling.h" -#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) -#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) -#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) -#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_SECT_NAME) -#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_SECT_NAME) -#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_SECT_NAME) +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) +#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) +#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) /* Declare section start and stop symbols for various sections * generated by compiler instrumentation. @@ -30,6 +30,7 @@ extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY; @@ -37,11 +38,13 @@ extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY; /* Add dummy data to ensure the section is always created. */ __llvm_profile_data - __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME_STR); + __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME); uint64_t - __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME_STR); -char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME_STR); -ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME_STR); + __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME); +uint32_t + __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME); +char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME); +ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data * __llvm_profile_begin_data(void) { @@ -63,6 +66,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { + return &PROF_ORDERFILE_START; +} COMPILER_RT_VISIBILITY ValueProfNode * __llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformOther.c b/lib/profile/InstrProfilingPlatformOther.c index 7c2f14cfc..56c5d8378 100644 --- a/lib/profile/InstrProfilingPlatformOther.c +++ b/lib/profile/InstrProfilingPlatformOther.c @@ -1,16 +1,17 @@ /*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ -#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \ + !defined(_WIN32) #include <stdlib.h> +#include <stdio.h> #include "InstrProfiling.h" @@ -20,6 +21,7 @@ static const char *NamesFirst = NULL; static const char *NamesLast = NULL; static uint64_t *CountersFirst = NULL; static uint64_t *CountersLast = NULL; +static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { return A1 < A2 ? A1 : A2; @@ -81,6 +83,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +/* TODO: correctly set up OrderFileFirst. */ +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformWindows.c b/lib/profile/InstrProfilingPlatformWindows.c new file mode 100644 index 000000000..81b708bb2 --- /dev/null +++ b/lib/profile/InstrProfilingPlatformWindows.c @@ -0,0 +1,68 @@ +/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(_WIN32) + +#if defined(_MSC_VER) +/* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") +#pragma comment(linker, "/MERGE:.lprfv=.data") +#pragma comment(linker, "/MERGE:.lprfnd=.data") +/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find + * after the fact. + */ + +/* Allocate read-only section bounds. */ +#pragma section(".lprfn$A", read) +#pragma section(".lprfn$Z", read) + +/* Allocate read-write section bounds. */ +#pragma section(".lprfd$A", read, write) +#pragma section(".lprfd$Z", read, write) +#pragma section(".lprfc$A", read, write) +#pragma section(".lprfc$Z", read, write) +#pragma section(".lorderfile$A", read, write) +#pragma section(".lprfnd$A", read, write) +#pragma section(".lprfnd$Z", read, write) +#endif + +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0}; +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; + +const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; +const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; + +uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; +uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; + +ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; +ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd; + +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart + 1; +} +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } + +const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } +const char *__llvm_profile_end_names(void) { return &NamesEnd; } + +uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +ValueProfNode *CurrentVNode = &VNodesStart + 1; +ValueProfNode *EndVNode = &VNodesEnd; + +#endif diff --git a/lib/profile/InstrProfilingPort.h b/lib/profile/InstrProfilingPort.h index ede6aaaf7..da5b5c0f8 100644 --- a/lib/profile/InstrProfilingPort.h +++ b/lib/profile/InstrProfilingPort.h @@ -1,9 +1,8 @@ /*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingRuntime.cc b/lib/profile/InstrProfilingRuntime.cc index eb8307498..679186ef8 100644 --- a/lib/profile/InstrProfilingRuntime.cc +++ b/lib/profile/InstrProfilingRuntime.cc @@ -1,9 +1,8 @@ //===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/profile/InstrProfilingUtil.c b/lib/profile/InstrProfilingUtil.c index 083bf14a3..5e479ae71 100644 --- a/lib/profile/InstrProfilingUtil.c +++ b/lib/profile/InstrProfilingUtil.c @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -81,7 +80,7 @@ void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { #endif -#ifdef _MSC_VER +#ifdef _WIN32 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; DWORD BufferSize = sizeof(Buffer); diff --git a/lib/profile/InstrProfilingUtil.h b/lib/profile/InstrProfilingUtil.h index 147677fc8..9cd0860fd 100644 --- a/lib/profile/InstrProfilingUtil.h +++ b/lib/profile/InstrProfilingUtil.h @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c index c7b01a570..b7c71768c 100644 --- a/lib/profile/InstrProfilingValue.c +++ b/lib/profile/InstrProfilingValue.c @@ -1,9 +1,8 @@ /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -32,7 +31,7 @@ static int hasNonDefaultValsPerSite = 0; * allocated by the compiler. */ COMPILER_RT_VISIBILITY ValueProfNode lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION( - COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME_STR); + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); #endif COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c index 7c6061d2c..d910cbb8f 100644 --- a/lib/profile/InstrProfilingWriter.c +++ b/lib/profile/InstrProfilingWriter.c @@ -1,9 +1,8 @@ /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/WindowsMMap.h b/lib/profile/WindowsMMap.h index 51a130b31..c8d6250f4 100644 --- a/lib/profile/WindowsMMap.h +++ b/lib/profile/WindowsMMap.h @@ -1,9 +1,8 @@ /*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/safestack/CMakeLists.txt b/lib/safestack/CMakeLists.txt index cc874a3fe..aa259e8d6 100644 --- a/lib/safestack/CMakeLists.txt +++ b/lib/safestack/CMakeLists.txt @@ -12,8 +12,6 @@ foreach(arch ${SAFESTACK_SUPPORTED_ARCH}) ARCHS ${arch} SOURCES ${SAFESTACK_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonNoLibc.${arch}> CFLAGS ${SAFESTACK_CFLAGS} PARENT_TARGET safestack) endforeach() diff --git a/lib/safestack/safestack.cc b/lib/safestack/safestack.cc index e68208015..f713d5e68 100644 --- a/lib/safestack/safestack.cc +++ b/lib/safestack/safestack.cc @@ -1,9 +1,8 @@ //===-- safestack.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,21 +13,31 @@ // //===----------------------------------------------------------------------===// +#include "safestack_platform.h" +#include "safestack_util.h" + #include <errno.h> -#include <limits.h> -#include <pthread.h> -#include <stddef.h> -#include <stdint.h> -#include <unistd.h> -#include <stdlib.h> #include <sys/resource.h> -#include <sys/types.h> -#if !defined(__NetBSD__) -#include <sys/user.h> -#endif #include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" + +using namespace safestack; + +// TODO: To make accessing the unsafe stack pointer faster, we plan to +// eventually store it directly in the thread control block data structure on +// platforms where this structure is pointed to by %fs or %gs. This is exactly +// the same mechanism as currently being used by the traditional stack +// protector pass to store the stack guard (see getStackCookieLocation() +// function above). Doing so requires changing the tcbhead_t struct in glibc +// on Linux and tcb struct in libc on FreeBSD. +// +// For now, store it in a thread-local variable. +extern "C" { +__attribute__((visibility( + "default"))) __thread void *__safestack_unsafe_stack_ptr = nullptr; +} + +namespace { // TODO: The runtime library does not currently protect the safe stack beyond // relying on the system-enforced ASLR. The protection of the (safe) stack can @@ -73,43 +82,26 @@ const unsigned kStackAlign = 16; /// size rlimit is set to infinity. const unsigned kDefaultUnsafeStackSize = 0x2800000; -/// Runtime page size obtained through sysconf -static unsigned pageSize; - -// TODO: To make accessing the unsafe stack pointer faster, we plan to -// eventually store it directly in the thread control block data structure on -// platforms where this structure is pointed to by %fs or %gs. This is exactly -// the same mechanism as currently being used by the traditional stack -// protector pass to store the stack guard (see getStackCookieLocation() -// function above). Doing so requires changing the tcbhead_t struct in glibc -// on Linux and tcb struct in libc on FreeBSD. -// -// For now, store it in a thread-local variable. -extern "C" { -__attribute__((visibility( - "default"))) __thread void *__safestack_unsafe_stack_ptr = nullptr; -} - // Per-thread unsafe stack information. It's not frequently accessed, so there // it can be kept out of the tcb in normal thread-local variables. -static __thread void *unsafe_stack_start = nullptr; -static __thread size_t unsafe_stack_size = 0; -static __thread size_t unsafe_stack_guard = 0; - -using namespace __sanitizer; - -static inline void *unsafe_stack_alloc(size_t size, size_t guard) { - CHECK_GE(size + guard, size); - void *addr = MmapOrDie(size + guard, "unsafe_stack_alloc"); - MprotectNoAccess((uptr)addr, (uptr)guard); +__thread void *unsafe_stack_start = nullptr; +__thread size_t unsafe_stack_size = 0; +__thread size_t unsafe_stack_guard = 0; + +inline void *unsafe_stack_alloc(size_t size, size_t guard) { + SFS_CHECK(size + guard >= size); + void *addr = Mmap(nullptr, size + guard, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + SFS_CHECK(MAP_FAILED != addr); + Mprotect(addr, guard, PROT_NONE); return (char *)addr + guard; } -static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { - CHECK_GE((char *)start + size, (char *)start); - CHECK_GE((char *)start + guard, (char *)start); +inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { + SFS_CHECK((char *)start + size >= (char *)start); + SFS_CHECK((char *)start + guard >= (char *)start); void *stack_ptr = (char *)start + size; - CHECK_EQ((((size_t)stack_ptr) & (kStackAlign - 1)), 0); + SFS_CHECK((((size_t)stack_ptr) & (kStackAlign - 1)) == 0); __safestack_unsafe_stack_ptr = stack_ptr; unsafe_stack_start = start; @@ -118,7 +110,7 @@ static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { } /// Thread data for the cleanup handler -static pthread_key_t thread_cleanup_key; +pthread_key_t thread_cleanup_key; /// Safe stack per-thread information passed to the thread_start function struct tinfo { @@ -132,7 +124,7 @@ struct tinfo { /// Wrap the thread function in order to deallocate the unsafe stack when the /// thread terminates by returning from its main function. -static void *thread_start(void *arg) { +void *thread_start(void *arg) { struct tinfo *tinfo = (struct tinfo *)arg; void *(*start_routine)(void *) = tinfo->start_routine; @@ -154,19 +146,19 @@ struct thread_stack_ll { void *stack_base; size_t size; pid_t pid; - tid_t tid; + ThreadId tid; }; /// Linked list of unsafe stacks for threads that are exiting. We delay /// unmapping them until the thread exits. -static thread_stack_ll *thread_stacks = nullptr; -static pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; +thread_stack_ll *thread_stacks = nullptr; +pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; /// Thread-specific data destructor. We want to free the unsafe stack only after /// this thread is terminated. libc can call functions in safestack-instrumented /// code (like free) after thread-specific data destructors have run. -static void thread_cleanup_handler(void *_iter) { - CHECK_NE(unsafe_stack_start, nullptr); +void thread_cleanup_handler(void *_iter) { + SFS_CHECK(unsafe_stack_start != nullptr); pthread_setspecific(thread_cleanup_key, NULL); pthread_mutex_lock(&thread_stacks_mutex); @@ -177,17 +169,15 @@ static void thread_cleanup_handler(void *_iter) { pthread_mutex_unlock(&thread_stacks_mutex); pid_t pid = getpid(); - tid_t tid = GetTid(); + ThreadId tid = GetTid(); // Free stacks for dead threads thread_stack_ll **stackp = &temp_stacks; while (*stackp) { thread_stack_ll *stack = *stackp; - int error; if (stack->pid != pid || - (internal_iserror(TgKill(stack->pid, stack->tid, 0), &error) && - error == ESRCH)) { - UnmapOrDie(stack->stack_base, stack->size); + (-1 == TgKill(stack->pid, stack->tid, 0) && errno == ESRCH)) { + Munmap(stack->stack_base, stack->size); *stackp = stack->next; free(stack); } else @@ -212,7 +202,7 @@ static void thread_cleanup_handler(void *_iter) { unsafe_stack_start = nullptr; } -static void EnsureInterceptorsInitialized(); +void EnsureInterceptorsInitialized(); /// Intercept thread creation operation to allocate and setup the unsafe stack INTERCEPTOR(int, pthread_create, pthread_t *thread, @@ -234,11 +224,12 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, pthread_attr_destroy(&tmpattr); } - CHECK_NE(size, 0); - CHECK_EQ((size & (kStackAlign - 1)), 0); - CHECK_EQ((guard & (pageSize - 1)), 0); + SFS_CHECK(size); + size = RoundUpTo(size, kStackAlign); void *addr = unsafe_stack_alloc(size, guard); + // Put tinfo at the end of the buffer. guard may be not page aligned. + // If that is so then some bytes after addr can be mprotected. struct tinfo *tinfo = (struct tinfo *)(((char *)addr) + size - sizeof(struct tinfo)); tinfo->start_routine = start_routine; @@ -250,12 +241,13 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, return REAL(pthread_create)(thread, attr, thread_start, tinfo); } -static BlockingMutex interceptor_init_lock(LINKER_INITIALIZED); -static bool interceptors_inited = false; +pthread_mutex_t interceptor_init_mutex = PTHREAD_MUTEX_INITIALIZER; +bool interceptors_inited = false; -static void EnsureInterceptorsInitialized() { - BlockingMutexLock lock(&interceptor_init_lock); - if (interceptors_inited) return; +void EnsureInterceptorsInitialized() { + MutexLock lock(interceptor_init_mutex); + if (interceptors_inited) + return; // Initialize pthread interceptors for thread allocation INTERCEPT_FUNCTION(pthread_create); @@ -263,6 +255,8 @@ static void EnsureInterceptorsInitialized() { interceptors_inited = true; } +} // namespace + extern "C" __attribute__((visibility("default"))) #if !SANITIZER_CAN_USE_PREINIT_ARRAY // On ELF platforms, the constructor is invoked using .preinit_array (see below) @@ -279,9 +273,7 @@ void __safestack_init() { // Allocate unsafe stack for main thread void *addr = unsafe_stack_alloc(size, guard); - unsafe_stack_setup(addr, size, guard); - pageSize = sysconf(_SC_PAGESIZE); // Setup the cleanup handler pthread_key_create(&thread_cleanup_key, thread_cleanup_handler); diff --git a/lib/safestack/safestack_platform.h b/lib/safestack/safestack_platform.h new file mode 100644 index 000000000..81e4c2645 --- /dev/null +++ b/lib/safestack/safestack_platform.h @@ -0,0 +1,124 @@ +//===-- safestack_platform.h ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements platform specific parts of SafeStack runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef SAFESTACK_PLATFORM_H +#define SAFESTACK_PLATFORM_H + +#include "safestack_util.h" +#include "sanitizer_common/sanitizer_platform.h" + +#include <dlfcn.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <unistd.h> + +#if !(SANITIZER_NETBSD || SANITIZER_FREEBSD || SANITIZER_LINUX) +#error "Support for your platform has not been implemented" +#endif + +#if SANITIZER_NETBSD +#include <lwp.h> + +extern "C" void *__mmap(void *, size_t, int, int, int, int, off_t); +#endif + +#if SANITIZER_FREEBSD +#include <sys/thr.h> +#endif + +namespace safestack { + +#if SANITIZER_NETBSD +static void *GetRealLibcAddress(const char *symbol) { + void *real = dlsym(RTLD_NEXT, symbol); + if (!real) + real = dlsym(RTLD_DEFAULT, symbol); + if (!real) { + fprintf(stderr, "safestack GetRealLibcAddress failed for symbol=%s", + symbol); + abort(); + } + return real; +} + +#define _REAL(func, ...) real##_##func(__VA_ARGS__) +#define DEFINE__REAL(ret_type, func, ...) \ + static ret_type (*real_##func)(__VA_ARGS__) = NULL; \ + if (!real_##func) { \ + real_##func = (ret_type(*)(__VA_ARGS__))GetRealLibcAddress(#func); \ + } \ + SFS_CHECK(real_##func); +#endif + +using ThreadId = uint64_t; + +inline ThreadId GetTid() { +#if SANITIZER_NETBSD + DEFINE__REAL(int, _lwp_self); + return _REAL(_lwp_self); +#elif SANITIZER_FREEBSD + long Tid; + thr_self(&Tid); + return Tid; +#else + return syscall(SYS_gettid); +#endif +} + +inline int TgKill(pid_t pid, ThreadId tid, int sig) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, _lwp_kill, int a, int b); + (void)pid; + return _REAL(_lwp_kill, tid, sig); +#elif SANITIZER_FREEBSD + return syscall(SYS_thr_kill2, pid, tid, sig); +#else + return syscall(SYS_tgkill, pid, tid, sig); +#endif +} + +inline void *Mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) { +#if SANITIZER_NETBSD + return __mmap(addr, length, prot, flags, fd, 0, offset); +#elif defined(__x86_64__) && (SANITIZER_FREEBSD) + return (void *)__syscall(SYS_mmap, addr, length, prot, flags, fd, offset); +#else + return (void *)syscall(SYS_mmap, addr, length, prot, flags, fd, offset); +#endif +} + +inline int Munmap(void *addr, size_t length) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, munmap, void *a, size_t b); + return _REAL(munmap, addr, length); +#else + return syscall(SYS_munmap, addr, length); +#endif +} + +inline int Mprotect(void *addr, size_t length, int prot) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, mprotect, void *a, size_t b, int c); + return _REAL(mprotect, addr, length, prot); +#else + return syscall(SYS_mprotect, addr, length, prot); +#endif +} + +} // namespace safestack + +#endif // SAFESTACK_PLATFORM_H diff --git a/lib/safestack/safestack_util.h b/lib/safestack/safestack_util.h new file mode 100644 index 000000000..da3f11b54 --- /dev/null +++ b/lib/safestack/safestack_util.h @@ -0,0 +1,49 @@ +//===-- safestack_util.h --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains utility code for SafeStack implementation. +// +//===----------------------------------------------------------------------===// + +#ifndef SAFESTACK_UTIL_H +#define SAFESTACK_UTIL_H + +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> + +namespace safestack { + +#define SFS_CHECK(a) \ + do { \ + if (!(a)) { \ + fprintf(stderr, "safestack CHECK failed: %s:%d %s\n", __FILE__, \ + __LINE__, #a); \ + abort(); \ + }; \ + } while (false) + +inline size_t RoundUpTo(size_t size, size_t boundary) { + SFS_CHECK((boundary & (boundary - 1)) == 0); + return (size + boundary - 1) & ~(boundary - 1); +} + +class MutexLock { + public: + explicit MutexLock(pthread_mutex_t &mutex) : mutex_(&mutex) { + pthread_mutex_lock(mutex_); + } + ~MutexLock() { pthread_mutex_unlock(mutex_); } + + private: + pthread_mutex_t *mutex_ = nullptr; +}; + +} // namespace safestack + +#endif // SAFESTACK_UTIL_H diff --git a/lib/sanitizer_common/CMakeLists.txt b/lib/sanitizer_common/CMakeLists.txt index f7bf4b009..75794163d 100644 --- a/lib/sanitizer_common/CMakeLists.txt +++ b/lib/sanitizer_common/CMakeLists.txt @@ -39,14 +39,8 @@ set(SANITIZER_SOURCES_NOTERMINATION sanitizer_tls_get_addr.cc sanitizer_thread_registry.cc sanitizer_type_traits.cc - sanitizer_win.cc) - -if(UNIX AND NOT APPLE AND NOT OS_NAME MATCHES "SunOS") - list(APPEND SANITIZER_SOURCES_NOTERMINATION - sanitizer_linux_x86_64.S) - list(APPEND SANITIZER_SOURCES_NOTERMINATION - sanitizer_linux_mips64.S) -endif() + sanitizer_win.cc + ) set(SANITIZER_SOURCES ${SANITIZER_SOURCES_NOTERMINATION} sanitizer_termination.cc) @@ -63,13 +57,15 @@ set(SANITIZER_LIBCDEP_SOURCES sanitizer_linux_libcdep.cc sanitizer_mac_libcdep.cc sanitizer_posix_libcdep.cc - sanitizer_stoptheworld_linux_libcdep.cc) + sanitizer_stoptheworld_linux_libcdep.cc + ) set(SANITIZER_COVERAGE_SOURCES sancov_flags.cc sanitizer_coverage_fuchsia.cc sanitizer_coverage_libcdep_new.cc - sanitizer_coverage_win_sections.cc) + sanitizer_coverage_win_sections.cc + ) set(SANITIZER_SYMBOLIZER_SOURCES sanitizer_allocator_report.cc @@ -87,7 +83,8 @@ set(SANITIZER_SYMBOLIZER_SOURCES sanitizer_symbolizer_report.cc sanitizer_symbolizer_win.cc sanitizer_unwind_linux_libcdep.cc - sanitizer_unwind_win.cc) + sanitizer_unwind_win.cc + ) # Explicitly list all sanitizer_common headers. Not all of these are # included in sanitizer_common source files, but we need to depend on @@ -138,6 +135,7 @@ set(SANITIZER_IMPL_HEADERS sanitizer_freebsd.h sanitizer_fuchsia.h sanitizer_getauxval.h + sanitizer_hash.h sanitizer_interceptors_ioctl_netbsd.inc sanitizer_interface_internal.h sanitizer_internal_defs.h @@ -188,7 +186,8 @@ set(SANITIZER_IMPL_HEADERS sanitizer_win.h sanitizer_win_defs.h sanitizer_win_dll_thunk.h - sanitizer_win_weak_interception.h) + sanitizer_win_weak_interception.h + ) include_directories(..) @@ -203,19 +202,6 @@ append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=570 append_list_if(COMPILER_RT_HAS_WGLOBAL_CONSTRUCTORS_FLAG -Wglobal-constructors SANITIZER_CFLAGS) -if (LLVM_ENABLE_PEDANTIC AND UNIX AND NOT APPLE) - # With -pedantic, our .S files raise warnings about empty macro arguments - # from __USER_LABEL_PREFIX__ being an empty arg to GLUE(). Unfortunately, - # there is no simple way to test for an empty define, nor to disable just - # that warning or to disable -pedantic. There is also no simple way to - # remove -pedantic from just this file (we'd have to remove from - # CMAKE_C*_FLAGS and re-add as a source property to all the non-.S files). - set_source_files_properties(sanitizer_linux_x86_64.S - PROPERTIES COMPILE_FLAGS "-w") - set_source_files_properties(sanitizer_linux_mips64.S - PROPERTIES COMPILE_FLAGS "-w") -endif () - if(APPLE) set(OS_OPTION OS ${SANITIZER_COMMON_SUPPORTED_OS}) endif() diff --git a/lib/sanitizer_common/sancov_flags.cc b/lib/sanitizer_common/sancov_flags.cc index 9abb5b5c5..db065519b 100644 --- a/lib/sanitizer_common/sancov_flags.cc +++ b/lib/sanitizer_common/sancov_flags.cc @@ -1,9 +1,8 @@ //===-- sancov_flags.cc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sancov_flags.h b/lib/sanitizer_common/sancov_flags.h index 627d9a3df..95d4ee5ca 100644 --- a/lib/sanitizer_common/sancov_flags.h +++ b/lib/sanitizer_common/sancov_flags.h @@ -1,9 +1,8 @@ //===-- sancov_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sancov_flags.inc b/lib/sanitizer_common/sancov_flags.inc index 63a1f0cbc..cca33fc35 100644 --- a/lib/sanitizer_common/sancov_flags.inc +++ b/lib/sanitizer_common/sancov_flags.inc @@ -1,9 +1,8 @@ //===-- sancov_flags.inc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_addrhashmap.h b/lib/sanitizer_common/sanitizer_addrhashmap.h index 2ca3c405b..a033e788c 100644 --- a/lib/sanitizer_common/sanitizer_addrhashmap.h +++ b/lib/sanitizer_common/sanitizer_addrhashmap.h @@ -1,9 +1,8 @@ //===-- sanitizer_addrhashmap.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator.cc b/lib/sanitizer_common/sanitizer_allocator.cc index 6bfd5e5ee..1739bb66b 100644 --- a/lib/sanitizer_common/sanitizer_allocator.cc +++ b/lib/sanitizer_common/sanitizer_allocator.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -171,6 +170,18 @@ void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { return (char*)p + sizeof(u64); } +void *InternalReallocArray(void *addr, uptr count, uptr size, + InternalAllocatorCache *cache) { + if (UNLIKELY(CheckForCallocOverflow(count, size))) { + Report( + "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) " + "cannot be represented in type size_t\n", + SanitizerToolName, count, size); + Die(); + } + return InternalRealloc(addr, count * size, cache); +} + void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { if (UNLIKELY(CheckForCallocOverflow(count, size))) { Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " diff --git a/lib/sanitizer_common/sanitizer_allocator.h b/lib/sanitizer_common/sanitizer_allocator.h index 88017160a..23d589888 100644 --- a/lib/sanitizer_common/sanitizer_allocator.h +++ b/lib/sanitizer_common/sanitizer_allocator.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_bytemap.h b/lib/sanitizer_common/sanitizer_allocator_bytemap.h index ef26941fe..0084bb62c 100644 --- a/lib/sanitizer_common/sanitizer_allocator_bytemap.h +++ b/lib/sanitizer_common/sanitizer_allocator_bytemap.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_bytemap.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_checks.cc b/lib/sanitizer_common/sanitizer_allocator_checks.cc index dc263dbef..bb56010f1 100644 --- a/lib/sanitizer_common/sanitizer_allocator_checks.cc +++ b/lib/sanitizer_common/sanitizer_allocator_checks.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_checks.h b/lib/sanitizer_common/sanitizer_allocator_checks.h index 61bd26e68..f436ce9ec 100644 --- a/lib/sanitizer_common/sanitizer_allocator_checks.h +++ b/lib/sanitizer_common/sanitizer_allocator_checks.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_checks.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_combined.h b/lib/sanitizer_common/sanitizer_allocator_combined.h index fcc4469c9..33f89d6d4 100644 --- a/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_combined.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,18 +19,15 @@ // When allocating 2^x bytes it should return 2^x aligned chunk. // PrimaryAllocator is used via a local AllocatorCache. // SecondaryAllocator can allocate anything, but is not efficient. -template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator, - typename AddressSpaceViewTy = LocalAddressSpaceView> // NOLINT +template <class PrimaryAllocator, + class LargeMmapAllocatorPtrArray = DefaultLargeMmapAllocatorPtrArray> class CombinedAllocator { public: - using AddressSpaceView = AddressSpaceViewTy; - static_assert(is_same<AddressSpaceView, - typename PrimaryAllocator::AddressSpaceView>::value, - "PrimaryAllocator is using wrong AddressSpaceView"); - static_assert(is_same<AddressSpaceView, - typename SecondaryAllocator::AddressSpaceView>::value, - "SecondaryAllocator is using wrong AddressSpaceView"); + using AllocatorCache = typename PrimaryAllocator::AllocatorCache; + using SecondaryAllocator = + LargeMmapAllocator<typename PrimaryAllocator::MapUnmapCallback, + LargeMmapAllocatorPtrArray, + typename PrimaryAllocator::AddressSpaceView>; void InitLinkerInitialized(s32 release_to_os_interval_ms) { stats_.InitLinkerInitialized(); diff --git a/lib/sanitizer_common/sanitizer_allocator_interface.h b/lib/sanitizer_common/sanitizer_allocator_interface.h index 2f5ce3151..c1b27563e 100644 --- a/lib/sanitizer_common/sanitizer_allocator_interface.h +++ b/lib/sanitizer_common/sanitizer_allocator_interface.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_interface.h ------------------------- C++ -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_internal.h b/lib/sanitizer_common/sanitizer_allocator_internal.h index 30fc7042b..32849036f 100644 --- a/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_internal.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,42 +22,30 @@ namespace __sanitizer { // purposes. typedef CompactSizeClassMap InternalSizeClassMap; -static const uptr kInternalAllocatorRegionSizeLog = 20; -static const uptr kInternalAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kInternalAllocatorRegionSizeLog; -#if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap<kInternalAllocatorNumRegions> ByteMap; -#else -typedef TwoLevelByteMap<(kInternalAllocatorNumRegions >> 12), 1 << 12> ByteMap; -#endif struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef InternalSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kInternalAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __sanitizer::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; typedef SizeClassAllocator32<AP32> PrimaryInternalAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator> - InternalAllocatorCache; - -typedef LargeMmapAllocator<NoOpMapUnmapCallback, - LargeMmapAllocatorPtrArrayStatic> - SecondaryInternalAllocator; - -typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache, - SecondaryInternalAllocator> InternalAllocator; +typedef CombinedAllocator<PrimaryInternalAllocator, + LargeMmapAllocatorPtrArrayStatic> + InternalAllocator; +typedef InternalAllocator::AllocatorCache InternalAllocatorCache; void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr, uptr alignment = 0); void *InternalRealloc(void *p, uptr size, InternalAllocatorCache *cache = nullptr); -void *InternalCalloc(uptr countr, uptr size, +void *InternalReallocArray(void *p, uptr count, uptr size, + InternalAllocatorCache *cache = nullptr); +void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); diff --git a/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/lib/sanitizer_common/sanitizer_allocator_local_cache.h index 1bb8fc27d..108dfc231 100644 --- a/lib/sanitizer_common/sanitizer_allocator_local_cache.h +++ b/lib/sanitizer_common/sanitizer_allocator_local_cache.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_local_cache.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,13 +13,6 @@ #error This file must be included inside sanitizer_allocator.h #endif -// Objects of this type should be used as local caches for SizeClassAllocator64 -// or SizeClassAllocator32. Since the typical use of this class is to have one -// object per thread in TLS, is has to be POD. -template<class SizeClassAllocator> -struct SizeClassAllocatorLocalCache - : SizeClassAllocator::AllocatorCache {}; - // Cache used by SizeClassAllocator64. template <class SizeClassAllocator> struct SizeClassAllocator64LocalCache { diff --git a/lib/sanitizer_common/sanitizer_allocator_primary32.h b/lib/sanitizer_common/sanitizer_allocator_primary32.h index abaac3d1a..3b1838b39 100644 --- a/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_primary32.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,6 +46,11 @@ struct SizeClassAllocator32FlagMasks { // Bit masks. template <class Params> class SizeClassAllocator32 { + private: + static const u64 kTwoLevelByteMapSize1 = + (Params::kSpaceSize >> Params::kRegionSizeLog) >> 12; + static const u64 kMinFirstMapSizeTwoLevelByteMap = 4; + public: using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; @@ -54,12 +58,15 @@ class SizeClassAllocator32 { static const uptr kMetadataSize = Params::kMetadataSize; typedef typename Params::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = Params::kRegionSizeLog; - typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; + using ByteMap = typename conditional< + (kTwoLevelByteMapSize1 < kMinFirstMapSizeTwoLevelByteMap), + FlatByteMap<(Params::kSpaceSize >> Params::kRegionSizeLog), + AddressSpaceView>, + TwoLevelByteMap<kTwoLevelByteMapSize1, 1 << 12, AddressSpaceView>>::type; - static_assert( - is_same<typename ByteMap::AddressSpaceView, AddressSpaceView>::value, - "AddressSpaceView type mismatch"); + COMPILER_CHECK(!SANITIZER_SIGN_EXTENDED_ADDRESSES || + (kSpaceSize & (kSpaceSize - 1)) == 0); static const bool kRandomShuffleChunks = Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; @@ -182,6 +189,8 @@ class SizeClassAllocator32 { bool PointerIsMine(const void *p) { uptr mem = reinterpret_cast<uptr>(p); + if (SANITIZER_SIGN_EXTENDED_ADDRESSES) + mem &= (kSpaceSize - 1); if (mem < kSpaceBeg || mem >= kSpaceBeg + kSpaceSize) return false; return GetSizeClass(p) != 0; @@ -207,7 +216,7 @@ class SizeClassAllocator32 { return ClassIdToSize(GetSizeClass(p)); } - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } uptr TotalMemoryUsed() { // No need to lock here. @@ -273,7 +282,9 @@ class SizeClassAllocator32 { }; COMPILER_CHECK(sizeof(SizeClassInfo) % kCacheLineSize == 0); - uptr ComputeRegionId(uptr mem) { + uptr ComputeRegionId(uptr mem) const { + if (SANITIZER_SIGN_EXTENDED_ADDRESSES) + mem &= (kSpaceSize - 1); const uptr res = mem >> kRegionSizeLog; CHECK_LT(res, kNumPossibleRegions); return res; diff --git a/lib/sanitizer_common/sanitizer_allocator_primary64.h b/lib/sanitizer_common/sanitizer_allocator_primary64.h index b063bf0d3..90603280e 100644 --- a/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_primary64.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -81,7 +80,8 @@ class SizeClassAllocator64 { CHECK_NE(NonConstSpaceBeg, ~(uptr)0); } SetReleaseToOSIntervalMs(release_to_os_interval_ms); - MapWithCallbackOrDie(SpaceEnd(), AdditionalSize()); + MapWithCallbackOrDie(SpaceEnd(), AdditionalSize(), + "SizeClassAllocator: region info"); // Check that the RegionInfo array is aligned on the CacheLine size. DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0); } @@ -154,7 +154,7 @@ class SizeClassAllocator64 { return true; } - bool PointerIsMine(const void *p) { + bool PointerIsMine(const void *p) const { uptr P = reinterpret_cast<uptr>(p); if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0) return P / kSpaceSize == kSpaceBeg / kSpaceSize; @@ -189,7 +189,7 @@ class SizeClassAllocator64 { uptr beg = chunk_idx * size; uptr next_beg = beg + size; if (class_id >= kNumClasses) return nullptr; - RegionInfo *region = GetRegionInfo(class_id); + const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id)); if (region->mapped_user >= next_beg) return reinterpret_cast<void*>(reg_beg + beg); return nullptr; @@ -200,7 +200,7 @@ class SizeClassAllocator64 { return ClassIdToSize(GetSizeClass(p)); } - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } void *GetMetaData(const void *p) { uptr class_id = GetSizeClass(p); @@ -634,8 +634,8 @@ class SizeClassAllocator64 { return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg)); } - bool MapWithCallback(uptr beg, uptr size) { - uptr mapped = address_range.Map(beg, size); + bool MapWithCallback(uptr beg, uptr size, const char *name) { + uptr mapped = address_range.Map(beg, size, name); if (UNLIKELY(!mapped)) return false; CHECK_EQ(beg, mapped); @@ -643,8 +643,8 @@ class SizeClassAllocator64 { return true; } - void MapWithCallbackOrDie(uptr beg, uptr size) { - CHECK_EQ(beg, address_range.MapOrDie(beg, size)); + void MapWithCallbackOrDie(uptr beg, uptr size, const char *name) { + CHECK_EQ(beg, address_range.MapOrDie(beg, size, name)); MapUnmapCallback().OnMap(beg, size); } @@ -662,7 +662,8 @@ class SizeClassAllocator64 { uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) + region->mapped_free_array; uptr new_map_size = new_mapped_free_array - region->mapped_free_array; - if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size))) + if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size, + "SizeClassAllocator: freearray"))) return false; region->mapped_free_array = new_mapped_free_array; } @@ -713,7 +714,8 @@ class SizeClassAllocator64 { if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size))) return false; if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user, - user_map_size))) + user_map_size, + "SizeClassAllocator: region data"))) return false; stat->Add(AllocatorStatMapped, user_map_size); region->mapped_user += user_map_size; @@ -733,7 +735,7 @@ class SizeClassAllocator64 { return false; if (UNLIKELY(!MapWithCallback( GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size, - meta_map_size))) + meta_map_size, "SizeClassAllocator: region metadata"))) return false; region->mapped_meta += meta_map_size; } diff --git a/lib/sanitizer_common/sanitizer_allocator_report.cc b/lib/sanitizer_common/sanitizer_allocator_report.cc index e93f90c2a..dfc418166 100644 --- a/lib/sanitizer_common/sanitizer_allocator_report.cc +++ b/lib/sanitizer_common/sanitizer_allocator_report.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_report.cc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -52,6 +51,18 @@ void NORETURN ReportCallocOverflow(uptr count, uptr size, Die(); } +void NORETURN ReportReallocArrayOverflow(uptr count, uptr size, + const StackTrace *stack) { + { + ScopedAllocatorErrorReport report("reallocarray-overflow", stack); + Report( + "ERROR: %s: reallocarray parameters overflow: count * size (%zd * %zd) " + "cannot be represented in type size_t\n", + SanitizerToolName, count, size); + } + Die(); +} + void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack) { { ScopedAllocatorErrorReport report("pvalloc-overflow", stack); diff --git a/lib/sanitizer_common/sanitizer_allocator_report.h b/lib/sanitizer_common/sanitizer_allocator_report.h index b19b22fa5..0653c365c 100644 --- a/lib/sanitizer_common/sanitizer_allocator_report.h +++ b/lib/sanitizer_common/sanitizer_allocator_report.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_report.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -22,6 +21,8 @@ namespace __sanitizer { void NORETURN ReportCallocOverflow(uptr count, uptr size, const StackTrace *stack); +void NORETURN ReportReallocArrayOverflow(uptr count, uptr size, + const StackTrace *stack); void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack); void NORETURN ReportInvalidAllocationAlignment(uptr alignment, const StackTrace *stack); diff --git a/lib/sanitizer_common/sanitizer_allocator_secondary.h b/lib/sanitizer_common/sanitizer_allocator_secondary.h index 0c8505c34..1d128f55d 100644 --- a/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ b/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_secondary.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,22 +183,25 @@ class LargeMmapAllocator { uptr p = reinterpret_cast<uptr>(ptr); SpinMutexLock l(&mutex_); uptr nearest_chunk = 0; + Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); // Cache-friendly linear search. for (uptr i = 0; i < n_chunks_; i++) { - uptr ch = reinterpret_cast<uptr>(chunks_[i]); + uptr ch = reinterpret_cast<uptr>(chunks[i]); if (p < ch) continue; // p is at left to this chunk, skip it. if (p - ch < p - nearest_chunk) nearest_chunk = ch; } if (!nearest_chunk) return nullptr; - Header *h = reinterpret_cast<Header *>(nearest_chunk); + const Header *h = + AddressSpaceView::Load(reinterpret_cast<Header *>(nearest_chunk)); + Header *h_ptr = reinterpret_cast<Header *>(nearest_chunk); CHECK_GE(nearest_chunk, h->map_beg); CHECK_LT(nearest_chunk, h->map_beg + h->map_size); CHECK_LE(nearest_chunk, p); if (h->map_beg + h->map_size <= p) return nullptr; - return GetUser(h); + return GetUser(h_ptr); } void EnsureSortedChunks() { @@ -219,9 +221,10 @@ class LargeMmapAllocator { uptr n = n_chunks_; if (!n) return nullptr; EnsureSortedChunks(); - auto min_mmap_ = reinterpret_cast<uptr>(chunks_[0]); - auto max_mmap_ = - reinterpret_cast<uptr>(chunks_[n - 1]) + chunks_[n - 1]->map_size; + Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); + auto min_mmap_ = reinterpret_cast<uptr>(chunks[0]); + auto max_mmap_ = reinterpret_cast<uptr>(chunks[n - 1]) + + AddressSpaceView::Load(chunks[n - 1])->map_size; if (p < min_mmap_ || p >= max_mmap_) return nullptr; uptr beg = 0, end = n - 1; @@ -229,23 +232,24 @@ class LargeMmapAllocator { // to avoid expensive cache-thrashing loads. while (end - beg >= 2) { uptr mid = (beg + end) / 2; // Invariant: mid >= beg + 1 - if (p < reinterpret_cast<uptr>(chunks_[mid])) - end = mid - 1; // We are not interested in chunks_[mid]. + if (p < reinterpret_cast<uptr>(chunks[mid])) + end = mid - 1; // We are not interested in chunks[mid]. else - beg = mid; // chunks_[mid] may still be what we want. + beg = mid; // chunks[mid] may still be what we want. } if (beg < end) { CHECK_EQ(beg + 1, end); // There are 2 chunks left, choose one. - if (p >= reinterpret_cast<uptr>(chunks_[end])) + if (p >= reinterpret_cast<uptr>(chunks[end])) beg = end; } - Header *h = chunks_[beg]; + const Header *h = AddressSpaceView::Load(chunks[beg]); + Header *h_ptr = chunks[beg]; if (h->map_beg + h->map_size <= p || p < h->map_beg) return nullptr; - return GetUser(h); + return GetUser(h_ptr); } void PrintStats() { diff --git a/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 1c05fb8ff..12d8c8923 100644 --- a/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_size_class_map.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_stats.h b/lib/sanitizer_common/sanitizer_allocator_stats.h index 76df3ed8d..6f14e3863 100644 --- a/lib/sanitizer_common/sanitizer_allocator_stats.h +++ b/lib/sanitizer_common/sanitizer_allocator_stats.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_stats.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_asm.h b/lib/sanitizer_common/sanitizer_asm.h index 76a0bf714..184d118d9 100644 --- a/lib/sanitizer_common/sanitizer_asm.h +++ b/lib/sanitizer_common/sanitizer_asm.h @@ -1,9 +1,8 @@ //===-- sanitizer_asm.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,14 +44,23 @@ #if !defined(__APPLE__) # define ASM_HIDDEN(symbol) .hidden symbol -# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function +# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function # define ASM_SIZE(symbol) .size symbol, .-symbol # define ASM_SYMBOL(symbol) symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) # define ASM_SIZE(symbol) # define ASM_SYMBOL(symbol) _##symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +#endif + +#if defined(__ELF__) && (defined(__GNU__) || defined(__FreeBSD__) || \ + defined(__Fuchsia__) || defined(__linux__)) +#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits // NOLINT +#else +#define NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/sanitizer_common/sanitizer_atomic.h b/lib/sanitizer_common/sanitizer_atomic.h index 8f400acc9..a798a0cf2 100644 --- a/lib/sanitizer_common/sanitizer_atomic.h +++ b/lib/sanitizer_common/sanitizer_atomic.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang.h b/lib/sanitizer_common/sanitizer_atomic_clang.h index cd41c920f..c40461ebc 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/lib/sanitizer_common/sanitizer_atomic_clang_mips.h index c64e6e542..d369aeb99 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_mips.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_mips.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/lib/sanitizer_common/sanitizer_atomic_clang_other.h index 2eea549c0..b8685a854 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_other.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_other.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/lib/sanitizer_common/sanitizer_atomic_clang_x86.h index 195533ea2..f2ce553ba 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_x86.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_msvc.h b/lib/sanitizer_common/sanitizer_atomic_msvc.h index 6d94056d8..a249657d6 100644 --- a/lib/sanitizer_common/sanitizer_atomic_msvc.h +++ b/lib/sanitizer_common/sanitizer_atomic_msvc.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_bitvector.h b/lib/sanitizer_common/sanitizer_bitvector.h index 0f65814c3..07a59ab11 100644 --- a/lib/sanitizer_common/sanitizer_bitvector.h +++ b/lib/sanitizer_common/sanitizer_bitvector.h @@ -1,9 +1,8 @@ //===-- sanitizer_bitvector.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_bvgraph.h b/lib/sanitizer_common/sanitizer_bvgraph.h index 9c2db9ae4..e7249055b 100644 --- a/lib/sanitizer_common/sanitizer_bvgraph.h +++ b/lib/sanitizer_common/sanitizer_bvgraph.h @@ -1,9 +1,8 @@ //===-- sanitizer_bvgraph.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common.cc b/lib/sanitizer_common/sanitizer_common.cc index 686896190..80fb8f60f 100644 --- a/lib/sanitizer_common/sanitizer_common.cc +++ b/lib/sanitizer_common/sanitizer_common.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h index d0aebd994..1703899e3 100644 --- a/lib/sanitizer_common/sanitizer_common.h +++ b/lib/sanitizer_common/sanitizer_common.h @@ -1,9 +1,8 @@ //===-- sanitizer_common.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -102,10 +101,11 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type); bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr) WARN_UNUSED_RESULT; void *MmapNoReserveOrDie(uptr size, const char *mem_type); -void *MmapFixedOrDie(uptr fixed_addr, uptr size); +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name = nullptr); // Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in // that case returns nullptr. -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size); +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, + const char *name = nullptr); void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoAccess(uptr size); // Map aligned chunk of address space; size and alignment are powers of two. @@ -141,8 +141,8 @@ void RunFreeHooks(const void *ptr); class ReservedAddressRange { public: uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0); - uptr Map(uptr fixed_addr, uptr size); - uptr MapOrDie(uptr fixed_addr, uptr size); + uptr Map(uptr fixed_addr, uptr size, const char *name = nullptr); + uptr MapOrDie(uptr fixed_addr, uptr size, const char *name = nullptr); void Unmap(uptr addr, uptr size); void *base() const { return base_; } uptr size() const { return size_; } @@ -238,7 +238,6 @@ char **GetArgv(); char **GetEnviron(); void PrintCmdline(); bool StackSizeIsUnlimited(); -uptr GetStackSizeLimitInBytes(); void SetStackSizeLimitInBytes(uptr limit); bool AddressSpaceIsUnlimited(); void SetAddressSpaceUnlimited(); @@ -804,7 +803,13 @@ enum AndroidApiLevel { void WriteToSyslog(const char *buffer); -#if SANITIZER_MAC +#if defined(SANITIZER_WINDOWS) && defined(_MSC_VER) && !defined(__clang__) +#define SANITIZER_WIN_TRACE 1 +#else +#define SANITIZER_WIN_TRACE 0 +#endif + +#if SANITIZER_MAC || SANITIZER_WIN_TRACE void LogFullErrorReport(const char *buffer); #else INLINE void LogFullErrorReport(const char *buffer) {} @@ -818,7 +823,7 @@ INLINE void WriteOneLineToSyslog(const char *s) {} INLINE void LogMessageOnPrintf(const char *str) {} #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_WIN_TRACE // Initialize Android logging. Any writes before this are silently lost. void AndroidLogInit(); void SetAbortMessage(const char *); diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc index 50f783774..bdecf7b0f 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors.inc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -820,16 +819,14 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { #endif #if SANITIZER_INTERCEPT_MEMCMP - DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, const void *s1, const void *s2, uptr n, int result) -INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) - return internal_memcmp(a1, a2, size); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); +// Common code for `memcmp` and `bcmp`. +int MemcmpInterceptorCommon(void *ctx, + int (*real_fn)(const void *, const void *, uptr), + const void *a1, const void *a2, uptr size) { if (common_flags()->intercept_memcmp) { if (common_flags()->strict_memcmp) { // Check the entire regions even if the first bytes of the buffers are @@ -855,17 +852,39 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { return r; } } - int result = REAL(memcmp(a1, a2, size)); + int result = real_fn(a1, a2, size); CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1, a2, size, result); return result; } +INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memcmp(a1, a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); + return MemcmpInterceptorCommon(ctx, REAL(memcmp), a1, a2, size); +} + #define INIT_MEMCMP COMMON_INTERCEPT_FUNCTION(memcmp) #else #define INIT_MEMCMP #endif +#if SANITIZER_INTERCEPT_BCMP +INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memcmp(a1, a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, bcmp, a1, a2, size); + return MemcmpInterceptorCommon(ctx, REAL(bcmp), a1, a2, size); +} + +#define INIT_BCMP COMMON_INTERCEPT_FUNCTION(bcmp) +#else +#define INIT_BCMP +#endif + #if SANITIZER_INTERCEPT_MEMCHR INTERCEPTOR(void*, memchr, const void *s, int c, SIZE_T n) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) @@ -1818,61 +1837,53 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) { #define INIT_IOCTL #endif -#if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || \ - SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || \ - SANITIZER_INTERCEPT_GETPWENT_R || \ - SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS || \ - SANITIZER_INTERCEPT_FGETPWENT_R || \ - SANITIZER_INTERCEPT_FGETGRENT_R -static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { +#if SANITIZER_POSIX +UNUSED static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { if (pwd) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd)); if (pwd->pw_name) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_name, - REAL(strlen)(pwd->pw_name) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name, + REAL(strlen)(pwd->pw_name) + 1); if (pwd->pw_passwd) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_passwd, - REAL(strlen)(pwd->pw_passwd) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd, + REAL(strlen)(pwd->pw_passwd) + 1); #if !SANITIZER_ANDROID if (pwd->pw_gecos) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_gecos, - REAL(strlen)(pwd->pw_gecos) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos, + REAL(strlen)(pwd->pw_gecos) + 1); #endif -#if SANITIZER_MAC +#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD if (pwd->pw_class) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_class, - REAL(strlen)(pwd->pw_class) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class, + REAL(strlen)(pwd->pw_class) + 1); #endif if (pwd->pw_dir) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_dir, - REAL(strlen)(pwd->pw_dir) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir, + REAL(strlen)(pwd->pw_dir) + 1); if (pwd->pw_shell) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_shell, - REAL(strlen)(pwd->pw_shell) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell, + REAL(strlen)(pwd->pw_shell) + 1); } } -static void unpoison_group(void *ctx, __sanitizer_group *grp) { +UNUSED static void unpoison_group(void *ctx, __sanitizer_group *grp) { if (grp) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp)); if (grp->gr_name) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_name, - REAL(strlen)(grp->gr_name) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name, + REAL(strlen)(grp->gr_name) + 1); if (grp->gr_passwd) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_passwd, - REAL(strlen)(grp->gr_passwd) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd, + REAL(strlen)(grp->gr_passwd) + 1); char **p = grp->gr_mem; for (; *p; ++p) { - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*p, REAL(strlen)(*p) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1); } - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_mem, - (p - grp->gr_mem + 1) * sizeof(*p)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem, + (p - grp->gr_mem + 1) * sizeof(*p)); } } -#endif // SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || - // SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || - // SANITIZER_INTERCEPT_GETPWENT_R || - // SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS +#endif // SANITIZER_POSIX #if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { @@ -1881,14 +1892,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_passwd *res = REAL(getpwnam)(name); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwuid, uid); __sanitizer_passwd *res = REAL(getpwuid)(uid); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { @@ -1896,14 +1907,14 @@ INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name); COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_group *res = REAL(getgrnam)(name); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrgid, u32 gid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrgid, gid); __sanitizer_group *res = REAL(getgrgid)(gid); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } #define INIT_GETPWNAM_AND_FRIENDS \ @@ -1925,10 +1936,8 @@ INTERCEPTOR(int, getpwnam_r, const char *name, __sanitizer_passwd *pwd, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwnam_r)(name, pwd, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_passwd(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_passwd(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1940,10 +1949,8 @@ INTERCEPTOR(int, getpwuid_r, u32 uid, __sanitizer_passwd *pwd, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwuid_r)(uid, pwd, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_passwd(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_passwd(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1956,10 +1963,8 @@ INTERCEPTOR(int, getgrnam_r, const char *name, __sanitizer_group *grp, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrnam_r)(name, grp, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_group(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_group(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1971,10 +1976,8 @@ INTERCEPTOR(int, getgrgid_r, u32 gid, __sanitizer_group *grp, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrgid_r)(gid, grp, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_group(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_group(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1992,14 +1995,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwent, dummy); __sanitizer_passwd *res = REAL(getpwent)(dummy); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrent, dummy); __sanitizer_group *res = REAL(getgrent)(dummy); - if (res) unpoison_group(ctx, res);; + unpoison_group(ctx, res); return res; } #define INIT_GETPWENT \ @@ -2014,14 +2017,14 @@ INTERCEPTOR(__sanitizer_passwd *, fgetpwent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent, fp); __sanitizer_passwd *res = REAL(fgetpwent)(fp); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, fgetgrent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent, fp); __sanitizer_group *res = REAL(fgetgrent)(fp); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } #define INIT_FGETPWENT \ @@ -2040,10 +2043,8 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwent_r)(pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_passwd(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2055,10 +2056,8 @@ INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_group(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2078,10 +2077,8 @@ INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_passwd(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2100,10 +2097,8 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetgrent_r)(fp, pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_group(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -3529,13 +3524,16 @@ INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, wcrtomb, dest, src, ps); if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - SIZE_T res = REAL(wcrtomb)(dest, src, ps); - if (res != ((SIZE_T)-1) && dest) { - SIZE_T write_cnt = res; - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt); + + if (!dest) + return REAL(wcrtomb)(dest, src, ps); + + char local_dest[32]; + SIZE_T res = REAL(wcrtomb)(local_dest, src, ps); + if (res != ((SIZE_T)-1)) { + CHECK_LE(res, sizeof(local_dest)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res); + REAL(memcpy)(dest, local_dest, res); } return res; } @@ -3545,6 +3543,28 @@ INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { #define INIT_WCRTOMB #endif +#if SANITIZER_INTERCEPT_WCTOMB +INTERCEPTOR(int, wctomb, char *dest, wchar_t src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, wctomb, dest, src); + if (!dest) + return REAL(wctomb)(dest, src); + + char local_dest[32]; + int res = REAL(wctomb)(local_dest, src); + if (res != -1) { + CHECK_LE(res, sizeof(local_dest)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res); + REAL(memcpy)(dest, local_dest, res); + } + return res; +} + +#define INIT_WCTOMB COMMON_INTERCEPT_FUNCTION(wctomb); +#else +#define INIT_WCTOMB +#endif + #if SANITIZER_INTERCEPT_TCGETATTR INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) { void *ctx; @@ -4041,6 +4061,25 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, #define INIT_SIGPROCMASK #endif +#if SANITIZER_INTERCEPT_PTHREAD_SIGMASK +INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_sigmask, how, set, oldset); + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); + // FIXME: under ASan the call below may write to freed memory and corrupt + // its metadata. See + // https://github.com/google/sanitizers/issues/321. + int res = REAL(pthread_sigmask)(how, set, oldset); + if (!res && oldset) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); + return res; +} +#define INIT_PTHREAD_SIGMASK COMMON_INTERCEPT_FUNCTION(pthread_sigmask); +#else +#define INIT_PTHREAD_SIGMASK +#endif + #if SANITIZER_INTERCEPT_BACKTRACE INTERCEPTOR(int, backtrace, void **buffer, int size) { void *ctx; @@ -4724,6 +4763,20 @@ INTERCEPTOR(char *, tmpnam_r, char *s) { #define INIT_TMPNAM_R #endif +#if SANITIZER_INTERCEPT_TTYNAME +INTERCEPTOR(char *, ttyname, int fd) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd); + char *res = REAL(ttyname)(fd); + if (res != nullptr) + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + return res; +} +#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname); +#else +#define INIT_TTYNAME +#endif + #if SANITIZER_INTERCEPT_TTYNAME_R INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) { void *ctx; @@ -5501,12 +5554,21 @@ INTERCEPTOR(void *, __bzero, void *block, uptr size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } - #define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); #else #define INIT___BZERO #endif // SANITIZER_INTERCEPT___BZERO +#if SANITIZER_INTERCEPT_BZERO +INTERCEPTOR(void *, bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); +#else +#define INIT_BZERO +#endif // SANITIZER_INTERCEPT_BZERO + #if SANITIZER_INTERCEPT_FTIME INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { void *ctx; @@ -9498,6 +9560,7 @@ static void InitializeCommonInterceptors() { INIT_MEMCPY; INIT_MEMCHR; INIT_MEMCMP; + INIT_BCMP; INIT_MEMRCHR; INIT_MEMMEM; INIT_READ; @@ -9578,6 +9641,7 @@ static void InitializeCommonInterceptors() { INIT_WCSTOMBS; INIT_WCSNRTOMBS; INIT_WCRTOMB; + INIT_WCTOMB; INIT_TCGETATTR; INIT_REALPATH; INIT_CANONICALIZE_FILE_NAME; @@ -9599,6 +9663,7 @@ static void InitializeCommonInterceptors() { INIT_SIGSETOPS; INIT_SIGPENDING; INIT_SIGPROCMASK; + INIT_PTHREAD_SIGMASK; INIT_BACKTRACE; INIT__EXIT; INIT_PTHREAD_MUTEX_LOCK; @@ -9637,6 +9702,7 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_BARRIERATTR_GETPSHARED; INIT_TMPNAM; INIT_TMPNAM_R; + INIT_TTYNAME; INIT_TTYNAME_R; INIT_TEMPNAM; INIT_PTHREAD_SETNAME_NP; @@ -9662,6 +9728,7 @@ static void InitializeCommonInterceptors() { INIT_CAPGET; INIT_AEABI_MEM; INIT___BZERO; + INIT_BZERO; INIT_FTIME; INIT_XDR; INIT_TSEARCH; diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc index 5ebe5a6ba..bbbedda8f 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors_format.inc ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 2d633c173..490a04b21 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors_ioctl.inc -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S new file mode 100644 index 000000000..20f42f1ea --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S @@ -0,0 +1,43 @@ +#if defined(__aarch64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save x30 in the off-stack spill area. + stp xzr, x30, [sp, #-16]! + bl COMMON_INTERCEPTOR_SPILL_AREA + ldp xzr, x30, [sp], 16 + str x30, [x0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + adrp x0, _ZN14__interception10real_vforkE + ldr x0, [x0, :lo12:_ZN14__interception10real_vforkE] + blr x0 + + stp x0, xzr, [sp, #-16]! + cmp x0, #0 + b.eq .L_exit + + // x0 != 0 => parent process. Clear stack shadow. + add x0, sp, #16 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore x30. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr x30, [x0] + ldp x0, xzr, [sp], 16 + + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S new file mode 100644 index 000000000..780a9d46e --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S @@ -0,0 +1,49 @@ +#if defined(__arm__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save LR in the off-stack spill area. + push {r4, lr} + bl COMMON_INTERCEPTOR_SPILL_AREA + pop {r4, lr} + str lr, [r0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + ldr r0, .LCPI0_0 +.LPC0_0: + ldr r0, [pc, r0] + mov lr, pc + bx r0 + + push {r0, r4} + cmp r0, #0 + beq .L_exit + + // r0 != 0 => parent process. Clear stack shadow. + add r0, sp, #8 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore LR. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr lr, [r0] + pop {r0, r4} + + mov pc, lr + +.LCPI0_0: + .long _ZN14__interception10real_vforkE - (.LPC0_0+8) + +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S new file mode 100644 index 000000000..ed693819c --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S @@ -0,0 +1,63 @@ +#if defined(__i386__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Store return address in the spill area and tear down the stack frame. + sub $12, %esp + call COMMON_INTERCEPTOR_SPILL_AREA + mov 12(%esp), %ecx + mov %ecx, (%eax) + add $16, %esp + + call .L0$pb +.L0$pb: + pop %eax +.Ltmp0: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax + call *_ZN14__interception10real_vforkE@GOTOFF(%eax) + + // Restore the stack frame. + // 12(%esp) return address + // 8(%esp) spill %ebx + // 4(%esp) spill REAL(vfork) return value + // (%esp) call frame (arg0) for __*_handle_vfork + sub $16, %esp + mov %ebx, 8(%esp) + mov %eax, 4(%esp) + + // Form GOT address in %ebx. + call .L1$pb +.L1$pb: + pop %ebx +.Ltmp1: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx + + // Restore original return address. + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%eax), %ecx + mov %ecx, 12(%esp) + mov 4(%esp), %eax + + // Call handle_vfork in the parent process (%rax != 0). + test %eax, %eax + je .L_exit + + lea 16(%esp), %ecx + mov %ecx, (%esp) + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + mov 4(%esp), %eax + mov 8(%esp), %ebx + add $12, %esp + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S new file mode 100644 index 000000000..8147cdd09 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S @@ -0,0 +1,41 @@ +#if defined(__x86_64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Store return address in the spill area and tear down the stack frame. + push %rcx + call COMMON_INTERCEPTOR_SPILL_AREA + pop %rcx + pop %rdi + mov %rdi, (%rax) + + call *_ZN14__interception10real_vforkE(%rip) + + // Restore return address from the spill area. + push %rcx + push %rax + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%rax), %rdx + mov %rdx, 8(%rsp) + mov (%rsp), %rax + + // Call handle_vfork in the parent process (%rax != 0). + test %rax, %rax + je .L_exit + + lea 16(%rsp), %rdi + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + pop %rax + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interface.inc b/lib/sanitizer_common/sanitizer_common_interface.inc index 377580cb0..c72554973 100644 --- a/lib/sanitizer_common/sanitizer_common_interface.inc +++ b/lib/sanitizer_common/sanitizer_common_interface.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interface.inc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Common interface list. diff --git a/lib/sanitizer_common/sanitizer_common_interface_posix.inc b/lib/sanitizer_common/sanitizer_common_interface_posix.inc index bbc725a9d..38f953114 100644 --- a/lib/sanitizer_common/sanitizer_common_interface_posix.inc +++ b/lib/sanitizer_common/sanitizer_common_interface_posix.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interface_posix.inc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Common interface list only available for Posix systems. diff --git a/lib/sanitizer_common/sanitizer_common_libcdep.cc b/lib/sanitizer_common/sanitizer_common_libcdep.cc index 1c0995b2d..363eb4c14 100644 --- a/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_libcdep.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_nolibc.cc b/lib/sanitizer_common/sanitizer_common_nolibc.cc index 5d3229073..fdd858781 100644 --- a/lib/sanitizer_common/sanitizer_common_nolibc.cc +++ b/lib/sanitizer_common/sanitizer_common_nolibc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_nolibc.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_syscalls.inc b/lib/sanitizer_common/sanitizer_common_syscalls.inc index 469c8eb7e..00bb2aeef 100644 --- a/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_syscalls.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc index 9ff8798a5..2c7180122 100644 --- a/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc +++ b/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_fuchsia.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -133,7 +132,7 @@ class TracePcGuardController final { // The first sample goes at [1] to reserve [0] for the magic number. next_index_ = 1 + num_guards; - zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_); + zx_status_t status = _zx_vmo_create(DataSize(), ZX_VMO_RESIZABLE, &vmo_); CHECK_EQ(status, ZX_OK); // Give the VMO a name including our process KOID so it's easy to spot. diff --git a/lib/sanitizer_common/sanitizer_coverage_interface.inc b/lib/sanitizer_common/sanitizer_coverage_interface.inc index f909b74c9..7beeff7e8 100644 --- a/lib/sanitizer_common/sanitizer_coverage_interface.inc +++ b/lib/sanitizer_common/sanitizer_coverage_interface.inc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_interface.inc ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Coverage interface list. diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc index bea277587..9dbf2eb52 100644 --- a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc +++ b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_libcdep_new.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Coverage Controller for Trace PC Guard. diff --git a/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc b/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc index d5e459f2c..1f0f69dc4 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_dll_thunk.cc -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc b/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc index 988a2065f..b2db7fd97 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_dynamic_runtime_thunk.cc -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_coverage_win_sections.cc b/lib/sanitizer_common/sanitizer_coverage_win_sections.cc index 108f76eff..403d46c8c 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_sections.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_sections.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_sections.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,35 +26,40 @@ #include "sanitizer_platform.h" #if SANITIZER_WINDOWS #include <stdint.h> -extern "C" { -// The Guard array and counter array should both be merged into the .data -// section to reduce the number of PE sections However, because PCTable is -// constant it should be merged with the .rdata section. -#pragma section(".SCOV$GA", read, write) // NOLINT -// Use align(1) to avoid adding any padding that will mess up clients trying to -// determine the start and end of the array. -__declspec(allocate(".SCOV$GA")) __declspec(align(1)) uint64_t - __start___sancov_guards = 0; -#pragma section(".SCOV$GZ", read, write) // NOLINT -__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint64_t - __stop___sancov_guards = 0; +extern "C" { +// Use uint64_t so the linker won't need to add any padding if it tries to word +// align the start of the 8-bit counters array. The array will always start 8 +// bytes after __start_sancov_cntrs. #pragma section(".SCOV$CA", read, write) // NOLINT -__declspec(allocate(".SCOV$CA")) __declspec(align(1)) uint64_t - __start___sancov_cntrs = 0; +__declspec(allocate(".SCOV$CA")) uint64_t __start___sancov_cntrs = 0; + +// Even though we said not to align __stop__sancov_cntrs (using the "align" +// declspec), MSVC's linker may try to align the section, .SCOV$CZ, containing +// it. This can cause a mismatch between the number of PCs and counters since +// each PCTable element is 8 bytes (unlike counters which are 1 byte) so no +// padding would be added to align .SCOVP$Z, However, if .SCOV$CZ section is 1 +// byte, the linker won't try to align it on an 8-byte boundary, so use a +// uint8_t for __stop_sancov_cntrs. #pragma section(".SCOV$CZ", read, write) // NOLINT -__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint64_t +__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint8_t __stop___sancov_cntrs = 0; +#pragma section(".SCOV$GA", read, write) // NOLINT +__declspec(allocate(".SCOV$GA")) uint64_t __start___sancov_guards = 0; +#pragma section(".SCOV$GZ", read, write) // NOLINT +__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint8_t + __stop___sancov_guards = 0; + +// The guard array and counter array should both be merged into the .data +// section to reduce the number of PE sections. However, because PCTable is +// constant it should be merged with the .rdata section. #pragma comment(linker, "/MERGE:.SCOV=.data") -// Use uint64_t so there won't be any issues if the linker tries to word align -// the pc array. #pragma section(".SCOVP$A", read) // NOLINT -__declspec(allocate(".SCOVP$A")) __declspec(align(1)) uint64_t - __start___sancov_pcs = 0; +__declspec(allocate(".SCOVP$A")) uint64_t __start___sancov_pcs = 0; #pragma section(".SCOVP$Z", read) // NOLINT -__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint64_t +__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint8_t __stop___sancov_pcs = 0; #pragma comment(linker, "/MERGE:.SCOVP=.rdata") diff --git a/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc b/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc index 0926f460b..1fd10d5b8 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_weak_interception.cc -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Sanitizer Coverage when it implemented as a diff --git a/lib/sanitizer_common/sanitizer_dbghelp.h b/lib/sanitizer_common/sanitizer_dbghelp.h index 1689edbf9..00a539980 100644 --- a/lib/sanitizer_common/sanitizer_dbghelp.h +++ b/lib/sanitizer_common/sanitizer_dbghelp.h @@ -1,9 +1,8 @@ //===-- sanitizer_dbghelp.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector.h b/lib/sanitizer_common/sanitizer_deadlock_detector.h index 86d5743e9..b80cff460 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector.h +++ b/lib/sanitizer_common/sanitizer_deadlock_detector.h @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,8 +25,8 @@ #ifndef SANITIZER_DEADLOCK_DETECTOR_H #define SANITIZER_DEADLOCK_DETECTOR_H -#include "sanitizer_common.h" #include "sanitizer_bvgraph.h" +#include "sanitizer_common.h" namespace __sanitizer { @@ -58,7 +57,6 @@ class DeadlockDetectorTLS { // Returns true if this is the first (non-recursive) acquisition of this lock. bool addLock(uptr lock_id, uptr current_epoch, u32 stk) { - // Printf("addLock: %zx %zx stk %u\n", lock_id, current_epoch, stk); CHECK_EQ(epoch_, current_epoch); if (!bv_.setBit(lock_id)) { // The lock is already held by this thread, it must be recursive. @@ -84,7 +82,6 @@ class DeadlockDetectorTLS { } } } - // Printf("remLock: %zx %zx\n", lock_id, epoch_); if (!bv_.clearBit(lock_id)) return; // probably addLock happened before flush if (n_all_locks_) { @@ -158,7 +155,6 @@ class DeadlockDetector { if (!available_nodes_.empty()) return getAvailableNode(data); if (!recycled_nodes_.empty()) { - // Printf("recycling: n_edges_ %zd\n", n_edges_); for (sptr i = n_edges_ - 1; i >= 0; i--) { if (recycled_nodes_.getBit(edges_[i].from) || recycled_nodes_.getBit(edges_[i].to)) { @@ -255,8 +251,6 @@ class DeadlockDetector { unique_tid}; edges_[n_edges_++] = e; } - // Printf("Edge%zd: %u %zd=>%zd in T%d\n", - // n_edges_, stk, added_edges[i], cur_idx, unique_tid); } return n_added_edges; } diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc index 68a99d2c6..a151a190c 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc +++ b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector1.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector2.cc b/lib/sanitizer_common/sanitizer_deadlock_detector2.cc index 0b0085a48..ed9ea26d3 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector2.cc +++ b/lib/sanitizer_common/sanitizer_deadlock_detector2.cc @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector2.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h index 11674dff5..a4722b080 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h +++ b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector_interface.h -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno.cc b/lib/sanitizer_common/sanitizer_errno.cc index a6f9fc612..1600de566 100644 --- a/lib/sanitizer_common/sanitizer_errno.cc +++ b/lib/sanitizer_common/sanitizer_errno.cc @@ -1,9 +1,8 @@ //===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno.h b/lib/sanitizer_common/sanitizer_errno.h index 9322608aa..584e66e4a 100644 --- a/lib/sanitizer_common/sanitizer_errno.h +++ b/lib/sanitizer_common/sanitizer_errno.h @@ -1,9 +1,8 @@ //===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno_codes.h b/lib/sanitizer_common/sanitizer_errno_codes.h index dba774c5b..f388d0d36 100644 --- a/lib/sanitizer_common/sanitizer_errno_codes.h +++ b/lib/sanitizer_common/sanitizer_errno_codes.h @@ -1,9 +1,8 @@ //===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_file.cc b/lib/sanitizer_common/sanitizer_file.cc index 278d75c52..3cb4974bd 100644 --- a/lib/sanitizer_common/sanitizer_file.cc +++ b/lib/sanitizer_common/sanitizer_file.cc @@ -1,9 +1,8 @@ //===-- sanitizer_file.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_file.h b/lib/sanitizer_common/sanitizer_file.h index 52e6ef8fb..4a78a0e0a 100644 --- a/lib/sanitizer_common/sanitizer_file.h +++ b/lib/sanitizer_common/sanitizer_file.h @@ -1,9 +1,8 @@ //===-- sanitizer_file.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flag_parser.cc b/lib/sanitizer_common/sanitizer_flag_parser.cc index 67830b294..655b1e814 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.cc +++ b/lib/sanitizer_common/sanitizer_flag_parser.cc @@ -1,9 +1,8 @@ //===-- sanitizer_flag_parser.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flag_parser.h b/lib/sanitizer_common/sanitizer_flag_parser.h index 705bfe23b..ea8bfebed 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/lib/sanitizer_common/sanitizer_flag_parser.h @@ -1,9 +1,8 @@ //===-- sanitizer_flag_parser.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,9 @@ namespace __sanitizer { class FlagHandlerBase { public: virtual bool Parse(const char *value) { return false; } + + protected: + ~FlagHandlerBase() {}; }; template <typename T> @@ -97,6 +99,15 @@ inline bool FlagHandler<uptr>::Parse(const char *value) { return ok; } +template <> +inline bool FlagHandler<s64>::Parse(const char *value) { + const char *value_end; + *t_ = internal_simple_strtoll(value, &value_end, 10); + bool ok = *value_end == 0; + if (!ok) Printf("ERROR: Invalid value for s64 option: '%s'\n", value); + return ok; +} + class FlagParser { static const int kMaxFlags = 200; struct Flag { diff --git a/lib/sanitizer_common/sanitizer_flags.cc b/lib/sanitizer_common/sanitizer_flags.cc index 4631dfb71..c587b1884 100644 --- a/lib/sanitizer_common/sanitizer_flags.cc +++ b/lib/sanitizer_common/sanitizer_flags.cc @@ -1,9 +1,8 @@ //===-- sanitizer_flags.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flags.h b/lib/sanitizer_common/sanitizer_flags.h index c2ec29d16..8f5e987da 100644 --- a/lib/sanitizer_common/sanitizer_flags.h +++ b/lib/sanitizer_common/sanitizer_flags.h @@ -1,9 +1,8 @@ //===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flags.inc b/lib/sanitizer_common/sanitizer_flags.inc index b047741b9..7d592bdcb 100644 --- a/lib/sanitizer_common/sanitizer_flags.inc +++ b/lib/sanitizer_common/sanitizer_flags.inc @@ -1,9 +1,8 @@ //===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -219,9 +218,9 @@ COMMON_FLAG(bool, intercept_stat, true, COMMON_FLAG(bool, intercept_send, true, "If set, uses custom wrappers for send* functions " "to find more errors.") -COMMON_FLAG(bool, decorate_proc_maps, false, "If set, decorate sanitizer " - "mappings in /proc/self/maps with " - "user-readable names") +COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID, + "If set, decorate sanitizer mappings in /proc/self/maps with " + "user-readable names") COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool " "found an error") COMMON_FLAG( diff --git a/lib/sanitizer_common/sanitizer_freebsd.h b/lib/sanitizer_common/sanitizer_freebsd.h index c9bba8064..64cb21f1c 100644 --- a/lib/sanitizer_common/sanitizer_freebsd.h +++ b/lib/sanitizer_common/sanitizer_freebsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_freebsd.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_fuchsia.cc b/lib/sanitizer_common/sanitizer_fuchsia.cc index 2c259b6cd..9f6851ff0 100644 --- a/lib/sanitizer_common/sanitizer_fuchsia.cc +++ b/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -1,9 +1,8 @@ //===-- sanitizer_fuchsia.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -253,12 +252,14 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, return addr; } -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size) { +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size, + const char *name) { return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, false); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size) { +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size, + const char *name) { return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true); } diff --git a/lib/sanitizer_common/sanitizer_fuchsia.h b/lib/sanitizer_common/sanitizer_fuchsia.h index 18821b4fd..5a2ad32b4 100644 --- a/lib/sanitizer_common/sanitizer_fuchsia.h +++ b/lib/sanitizer_common/sanitizer_fuchsia.h @@ -1,9 +1,8 @@ //===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_getauxval.h b/lib/sanitizer_common/sanitizer_getauxval.h index b22fcade5..cbd1af12c 100644 --- a/lib/sanitizer_common/sanitizer_getauxval.h +++ b/lib/sanitizer_common/sanitizer_getauxval.h @@ -1,9 +1,8 @@ //===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_hash.h b/lib/sanitizer_common/sanitizer_hash.h new file mode 100644 index 000000000..3d97dcc5d --- /dev/null +++ b/lib/sanitizer_common/sanitizer_hash.h @@ -0,0 +1,43 @@ +//===-- sanitizer_common.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple hash function. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_HASH_H +#define SANITIZER_HASH_H + +#include "sanitizer_internal_defs.h" + +namespace __sanitizer { +class MurMur2HashBuilder { + static const u32 m = 0x5bd1e995; + static const u32 seed = 0x9747b28c; + static const u32 r = 24; + u32 h; + + public: + explicit MurMur2HashBuilder(u32 init = 0) { h = seed ^ init; } + void add(u32 k) { + k *= m; + k ^= k >> r; + k *= m; + h *= m; + h ^= k; + } + u32 get() { + u32 x = h; + x ^= x >> 13; + x *= m; + x ^= x >> 15; + return x; + } +}; +} //namespace __sanitizer + +#endif // SANITIZER_HASH_H diff --git a/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc index 86cb44047..f29226b3e 100644 --- a/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc +++ b/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc @@ -1,9 +1,8 @@ //===-- sanitizer_interceptors_ioctl_netbsd.inc -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ struct ioctl_desc { const char *name; }; -const unsigned ioctl_table_max = 1202; +const unsigned ioctl_table_max = 1200; static ioctl_desc ioctl_table[ioctl_table_max]; static unsigned ioctl_table_size = 0; @@ -298,9 +297,6 @@ static void ioctl_table_fill() { _(IRFRAMETTY_GET_DEVICE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_GET_DONGLE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_SET_DONGLE, READ, sizeof(unsigned int)); - /* Entries from file: dev/isa/satlinkio.h */ - _(SATIORESET, NONE, 0); - _(SATIOGID, WRITE, struct_satlink_id_sz); /* Entries from file: dev/isa/isvio.h */ _(ISV_CMD, READWRITE, struct_isv_cmd_sz); /* Entries from file: dev/isa/wtreg.h */ @@ -649,8 +645,8 @@ static void ioctl_table_fill() { _(SPKRTUNE, NONE, 0); _(SPKRGETVOL, WRITE, sizeof(unsigned int)); _(SPKRSETVOL, READ, sizeof(unsigned int)); - /* Entries from file: dev/nvmm/nvmm_ioctl.h */ #if 0 /* WIP */ + /* Entries from file: dev/nvmm/nvmm_ioctl.h */ _(NVMM_IOC_CAPABILITY, WRITE, struct_nvmm_ioc_capability_sz); _(NVMM_IOC_MACHINE_CREATE, READWRITE, struct_nvmm_ioc_machine_create_sz); _(NVMM_IOC_MACHINE_DESTROY, READ, struct_nvmm_ioc_machine_destroy_sz); @@ -659,7 +655,7 @@ static void ioctl_table_fill() { _(NVMM_IOC_VCPU_DESTROY, READ, struct_nvmm_ioc_vcpu_destroy_sz); _(NVMM_IOC_VCPU_SETSTATE, READ, struct_nvmm_ioc_vcpu_setstate_sz); _(NVMM_IOC_VCPU_GETSTATE, READ, struct_nvmm_ioc_vcpu_getstate_sz); - _(NVMM_IOC_VCPU_INJECT, READWRITE, struct_nvmm_ioc_vcpu_inject_sz); + _(NVMM_IOC_VCPU_INJECT, READ, struct_nvmm_ioc_vcpu_inject_sz); _(NVMM_IOC_VCPU_RUN, READWRITE, struct_nvmm_ioc_vcpu_run_sz); _(NVMM_IOC_GPA_MAP, READ, struct_nvmm_ioc_gpa_map_sz); _(NVMM_IOC_GPA_UNMAP, READ, struct_nvmm_ioc_gpa_unmap_sz); diff --git a/lib/sanitizer_common/sanitizer_interface_internal.h b/lib/sanitizer_common/sanitizer_interface_internal.h index f53d9557e..c110eff13 100644 --- a/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/lib/sanitizer_common/sanitizer_interface_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_interface_internal.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h index 14258d617..e0c6506be 100644 --- a/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/lib/sanitizer_common/sanitizer_internal_defs.h @@ -1,9 +1,8 @@ //===-- sanitizer_internal_defs.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -423,7 +422,6 @@ inline void Trap() { namespace __asan { using namespace __sanitizer; } // NOLINT namespace __dsan { using namespace __sanitizer; } // NOLINT namespace __dfsan { using namespace __sanitizer; } // NOLINT -namespace __esan { using namespace __sanitizer; } // NOLINT namespace __lsan { using namespace __sanitizer; } // NOLINT namespace __msan { using namespace __sanitizer; } // NOLINT namespace __hwasan { using namespace __sanitizer; } // NOLINT diff --git a/lib/sanitizer_common/sanitizer_lfstack.h b/lib/sanitizer_common/sanitizer_lfstack.h index 879cc8092..af2ca55ec 100644 --- a/lib/sanitizer_common/sanitizer_lfstack.h +++ b/lib/sanitizer_common/sanitizer_lfstack.h @@ -1,9 +1,8 @@ //===-- sanitizer_lfstack.h -=-----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libc.cc b/lib/sanitizer_common/sanitizer_libc.cc index 4032cb104..95c74441f 100644 --- a/lib/sanitizer_common/sanitizer_libc.cc +++ b/lib/sanitizer_common/sanitizer_libc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_libc.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libc.h b/lib/sanitizer_common/sanitizer_libc.h index 7ed4e28a5..3d5db35d6 100644 --- a/lib/sanitizer_common/sanitizer_libc.h +++ b/lib/sanitizer_common/sanitizer_libc.h @@ -1,9 +1,8 @@ //===-- sanitizer_libc.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libignore.cc b/lib/sanitizer_common/sanitizer_libignore.cc index 49c4ba429..6c7c132e9 100644 --- a/lib/sanitizer_common/sanitizer_libignore.cc +++ b/lib/sanitizer_common/sanitizer_libignore.cc @@ -1,9 +1,8 @@ //===-- sanitizer_libignore.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/sanitizer_common/sanitizer_libignore.h b/lib/sanitizer_common/sanitizer_libignore.h index 49967b1e8..256f68597 100644 --- a/lib/sanitizer_common/sanitizer_libignore.h +++ b/lib/sanitizer_common/sanitizer_libignore.h @@ -1,9 +1,8 @@ //===-- sanitizer_libignore.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc index 48795674c..88ab0979b 100644 --- a/lib/sanitizer_common/sanitizer_linux.cc +++ b/lib/sanitizer_common/sanitizer_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -129,12 +128,6 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0 #endif -#if defined(__x86_64__) || SANITIZER_MIPS64 -extern "C" { -extern void internal_sigreturn(); -} -#endif - // Note : FreeBSD had implemented both // Linux and OpenBSD apis, available from // future 12.x version most likely @@ -401,7 +394,7 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #else - return internal_syscall(SYSCALL(readlink), path, buf, bufsize); + return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize); #endif } @@ -838,24 +831,6 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { } return result; } - -// Invokes sigaction via a raw syscall with a restorer, but does not support -// all platforms yet. -// We disable for Go simply because we have not yet added to buildgo.sh. -#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO -int internal_sigaction_syscall(int signum, const void *act, void *oldact) { - if (act == nullptr) - return internal_sigaction_norestorer(signum, act, oldact); - __sanitizer_sigaction u_adjust; - internal_memcpy(&u_adjust, act, sizeof(u_adjust)); -#if !SANITIZER_ANDROID || !SANITIZER_MIPS32 - if (u_adjust.sa_restorer == nullptr) { - u_adjust.sa_restorer = internal_sigreturn; - } -#endif - return internal_sigaction_norestorer(signum, (const void *)&u_adjust, oldact); -} -#endif // defined(__x86_64__) && !SANITIZER_GO #endif // SANITIZER_LINUX uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, @@ -1055,6 +1030,8 @@ uptr GetMaxVirtualAddress() { return (1ULL << 40) - 1; // 0x000000ffffffffffUL; # elif defined(__s390x__) return (1ULL << 53) - 1; // 0x001fffffffffffffUL; +#elif defined(__sparc__) + return ~(uptr)0; # else return (1ULL << 47) - 1; // 0x00007fffffffffffUL; # endif @@ -1846,10 +1823,20 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { u64 esr; if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; return esr & ESR_ELx_WNR ? WRITE : READ; -#elif SANITIZER_SOLARIS && defined(__sparc__) +#elif defined(__sparc__) // Decode the instruction to determine the access type. // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype). +#if SANITIZER_SOLARIS uptr pc = ucontext->uc_mcontext.gregs[REG_PC]; +#else + // Historical BSDism here. + struct sigcontext *scontext = (struct sigcontext *)context; +#if defined(__arch64__) + uptr pc = scontext->sigc_regs.tpc; +#else + uptr pc = scontext->si_regs.pc; +#endif +#endif u32 instr = *(u32 *)pc; return (instr >> 21) & 1 ? WRITE: READ; #else @@ -1940,28 +1927,27 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { // pointer, but GCC always uses r31 when we need a frame pointer. *bp = ucontext->uc_mcontext.regs->gpr[PT_R31]; #elif defined(__sparc__) - ucontext_t *ucontext = (ucontext_t*)context; - uptr *stk_ptr; -# if defined(__sparcv9) || defined (__arch64__) -# ifndef MC_PC -# define MC_PC REG_PC -# endif -# ifndef MC_O6 -# define MC_O6 REG_O6 +#if defined(__arch64__) || defined(__sparcv9) +#define STACK_BIAS 2047 +#else +#define STACK_BIAS 0 # endif # if SANITIZER_SOLARIS -# define mc_gregs gregs -# endif - *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; - *sp = ucontext->uc_mcontext.mc_gregs[MC_O6]; - stk_ptr = (uptr *) (*sp + 2047); - *bp = stk_ptr[15]; -# else + ucontext_t *ucontext = (ucontext_t *)context; *pc = ucontext->uc_mcontext.gregs[REG_PC]; - *sp = ucontext->uc_mcontext.gregs[REG_O6]; - stk_ptr = (uptr *) *sp; - *bp = stk_ptr[15]; + *sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS; +#else + // Historical BSDism here. + struct sigcontext *scontext = (struct sigcontext *)context; +#if defined(__arch64__) + *pc = scontext->sigc_regs.tpc; + *sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS; +#else + *pc = scontext->si_regs.pc; + *sp = scontext->si_regs.u_regs[14]; +#endif # endif + *bp = (uptr)((uhwptr *)*sp)[14] + STACK_BIAS; #elif defined(__mips__) ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.pc; diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h index c309e33f8..d2186ef5a 100644 --- a/lib/sanitizer_common/sanitizer_linux.h +++ b/lib/sanitizer_common/sanitizer_linux.h @@ -1,9 +1,8 @@ //===-- sanitizer_linux.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -59,10 +58,6 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); // (like the process-wide error reporting SEGV handler) must use // internal_sigaction instead. int internal_sigaction_norestorer(int signum, const void *act, void *oldact); -#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO -// Uses a raw system call to avoid interceptors. -int internal_sigaction_syscall(int signum, const void *act, void *oldact); -#endif void internal_sigdelset(__sanitizer_sigset_t *set, int signum); #if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \ || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \ @@ -106,6 +101,17 @@ bool LibraryNameIs(const char *full_name, const char *base_name); // Call cb for each region mapped by map. void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)); +// Releases memory pages entirely within the [beg, end] address range. +// The pages no longer count toward RSS; reads are guaranteed to return 0. +// Requires (but does not verify!) that pages are MAP_PRIVATE. +INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) { + // man madvise on Linux promises zero-fill for anonymous private pages. + // Testing shows the same behaviour for private (but not anonymous) mappings + // of shm_open() files, as long as the underlying file is untouched. + CHECK(SANITIZER_LINUX); + ReleaseMemoryPagesToOS(beg, end); +} + #if SANITIZER_ANDROID #if defined(__aarch64__) diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 6ce47ec62..0608898a1 100644 --- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux_libcdep.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -53,6 +52,7 @@ #endif #if SANITIZER_SOLARIS +#include <stdlib.h> #include <thread.h> #endif diff --git a/lib/sanitizer_common/sanitizer_linux_mips64.S b/lib/sanitizer_common/sanitizer_linux_mips64.S deleted file mode 100644 index 8729642aa..000000000 --- a/lib/sanitizer_common/sanitizer_linux_mips64.S +++ /dev/null @@ -1,23 +0,0 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. - -// Avoid being marked as needing an executable stack: -#if defined(__linux__) && defined(__ELF__) -.section .note.GNU-stack,"",%progbits -#endif - -// Further contents are mips64 only: -#if defined(__linux__) && defined(__mips64) - -.section .text -.set noreorder -.globl internal_sigreturn -.type internal_sigreturn, @function -internal_sigreturn: - - li $v0,5211 // #5211 is for SYS_rt_sigreturn - syscall - -.size internal_sigreturn, .-internal_sigreturn - -#endif // defined(__linux__) && defined(__mips64) diff --git a/lib/sanitizer_common/sanitizer_linux_s390.cc b/lib/sanitizer_common/sanitizer_linux_s390.cc index ad8c87f2d..b681bef35 100644 --- a/lib/sanitizer_common/sanitizer_linux_s390.cc +++ b/lib/sanitizer_common/sanitizer_linux_s390.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux_s390.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_linux_x86_64.S b/lib/sanitizer_common/sanitizer_linux_x86_64.S deleted file mode 100644 index 8ff909542..000000000 --- a/lib/sanitizer_common/sanitizer_linux_x86_64.S +++ /dev/null @@ -1,25 +0,0 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. - -// Avoid being marked as needing an executable stack: -#if defined(__linux__) && defined(__ELF__) -.section .note.GNU-stack,"",%progbits -#endif - -// Further contents are x86_64-only: -#if defined(__linux__) && defined(__x86_64__) - -#include "../builtins/assembly.h" - -// If the "naked" function attribute were supported for x86 we could -// do this via inline asm. -.text -.balign 4 -DEFINE_COMPILERRT_FUNCTION(internal_sigreturn) - mov $0xf, %eax // 0xf == SYS_rt_sigreturn - mov %rcx, %r10 - syscall - ret // Won't normally reach here. -END_COMPILERRT_FUNCTION(internal_sigreturn) - -#endif // defined(__linux__) && defined(__x86_64__) diff --git a/lib/sanitizer_common/sanitizer_list.h b/lib/sanitizer_common/sanitizer_list.h index 598ce51d8..f0b925945 100644 --- a/lib/sanitizer_common/sanitizer_list.h +++ b/lib/sanitizer_common/sanitizer_list.h @@ -1,9 +1,8 @@ //===-- sanitizer_list.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_local_address_space_view.h b/lib/sanitizer_common/sanitizer_local_address_space_view.h index ec1847abc..5d1b5264b 100644 --- a/lib/sanitizer_common/sanitizer_local_address_space_view.h +++ b/lib/sanitizer_common/sanitizer_local_address_space_view.h @@ -1,9 +1,8 @@ //===-- sanitizer_local_address_space_view.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc index 8954a7a88..4ddc6d106 100644 --- a/lib/sanitizer_common/sanitizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_mac.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,25 +239,25 @@ int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, (size_t)newlen); } -int internal_forkpty(int *amaster) { - int master, slave; - if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; +int internal_forkpty(int *aparent) { + int parent, worker; + if (openpty(&parent, &worker, nullptr, nullptr, nullptr) == -1) return -1; int pid = internal_fork(); if (pid == -1) { - close(master); - close(slave); + close(parent); + close(worker); return -1; } if (pid == 0) { - close(master); - if (login_tty(slave) != 0) { + close(parent); + if (login_tty(worker) != 0) { // We already forked, there's not much we can do. Let's quit. Report("login_tty failed (errno %d)\n", errno); internal__exit(1); } } else { - *amaster = master; - close(slave); + *aparent = parent; + close(worker); } return pid; } diff --git a/lib/sanitizer_common/sanitizer_mac.h b/lib/sanitizer_common/sanitizer_mac.h index 52825f8cf..7ba394fb7 100644 --- a/lib/sanitizer_common/sanitizer_mac.h +++ b/lib/sanitizer_common/sanitizer_mac.h @@ -1,9 +1,8 @@ //===-- sanitizer_mac.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/lib/sanitizer_common/sanitizer_mac_libcdep.cc index 090f17c88..93fb5b2f9 100644 --- a/lib/sanitizer_common/sanitizer_mac_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_mac_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_mac_libcdep.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_malloc_mac.inc b/lib/sanitizer_common/sanitizer_malloc_mac.inc index 44c914cea..3f3581eeb 100644 --- a/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -1,9 +1,8 @@ //===-- sanitizer_malloc_mac.inc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,6 +39,8 @@ struct sanitizer_malloc_introspection_t : public malloc_introspection_t { // Used to track changes to the allocator that will affect // zone enumeration. u64 allocator_enumeration_version; + uptr allocator_ptr; + uptr allocator_size; }; u64 GetMallocZoneAllocatorEnumerationVersion() { @@ -281,13 +282,48 @@ void __sanitizer_mz_free_definite_size( } #endif -kern_return_t mi_enumerator(task_t task, void *, - unsigned type_mask, vm_address_t zone_address, - memory_reader_t reader, +#ifndef COMMON_MALLOC_HAS_ZONE_ENUMERATOR +#error "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be defined" +#endif +static_assert((COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 0 || + (COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 1, + "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be 0 or 1"); + +#if COMMON_MALLOC_HAS_ZONE_ENUMERATOR +// Forward declare and expect the implementation to provided by +// includer. +kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, + vm_range_recorder_t recorder); +#else +// Provide stub implementation that fails. +kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, vm_range_recorder_t recorder) { - // Should enumerate all the pointers we have. Seems like a lot of work. + // Not supported. return KERN_FAILURE; } +#endif + +#ifndef COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT +#error "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be defined" +#endif +static_assert((COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 0 || + (COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 1, + "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be 0 or 1"); +#if COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT +// Forward declare and expect the implementation to provided by +// includer. +void mi_extra_init( + sanitizer_malloc_introspection_t *mi); +#else +void mi_extra_init( + sanitizer_malloc_introspection_t *mi) { + // Just zero initialize the fields. + mi->allocator_ptr = 0; + mi->allocator_size = 0; +} +#endif size_t mi_good_size(malloc_zone_t *zone, size_t size) { // I think it's always safe to return size, but we maybe could do better. @@ -347,6 +383,9 @@ void InitMallocZoneFields() { sanitizer_zone_introspection.allocator_enumeration_version = GetMallocZoneAllocatorEnumerationVersion(); + // Perform any sanitizer specific initialization. + mi_extra_init(&sanitizer_zone_introspection); + internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t)); // Use version 6 for OSX >= 10.6. diff --git a/lib/sanitizer_common/sanitizer_mutex.h b/lib/sanitizer_common/sanitizer_mutex.h index a93705be6..40a659142 100644 --- a/lib/sanitizer_common/sanitizer_mutex.h +++ b/lib/sanitizer_common/sanitizer_mutex.h @@ -1,9 +1,8 @@ //===-- sanitizer_mutex.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_netbsd.cc b/lib/sanitizer_common/sanitizer_netbsd.cc index 80d0855ef..314b743cd 100644 --- a/lib/sanitizer_common/sanitizer_netbsd.cc +++ b/lib/sanitizer_common/sanitizer_netbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_netbsd.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_openbsd.cc b/lib/sanitizer_common/sanitizer_openbsd.cc index f0d071e51..b6e9bdfa8 100644 --- a/lib/sanitizer_common/sanitizer_openbsd.cc +++ b/lib/sanitizer_common/sanitizer_openbsd.cc @@ -1,7 +1,8 @@ //===-- sanitizer_openbsd.cc ----------------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_persistent_allocator.cc b/lib/sanitizer_common/sanitizer_persistent_allocator.cc index 5fa533a70..ddd9ae9ec 100644 --- a/lib/sanitizer_common/sanitizer_persistent_allocator.cc +++ b/lib/sanitizer_common/sanitizer_persistent_allocator.cc @@ -1,9 +1,8 @@ //===-- sanitizer_persistent_allocator.cc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_persistent_allocator.h b/lib/sanitizer_common/sanitizer_persistent_allocator.h index 8e5ce06d4..de4fb6ebc 100644 --- a/lib/sanitizer_common/sanitizer_persistent_allocator.h +++ b/lib/sanitizer_common/sanitizer_persistent_allocator.h @@ -1,9 +1,8 @@ //===-- sanitizer_persistent_allocator.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_placement_new.h b/lib/sanitizer_common/sanitizer_placement_new.h index 8904d1040..1ceb8b909 100644 --- a/lib/sanitizer_common/sanitizer_placement_new.h +++ b/lib/sanitizer_common/sanitizer_placement_new.h @@ -1,9 +1,8 @@ //===-- sanitizer_placement_new.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform.h b/lib/sanitizer_common/sanitizer_platform.h index 82bb1af77..b45c97535 100644 --- a/lib/sanitizer_common/sanitizer_platform.h +++ b/lib/sanitizer_common/sanitizer_platform.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -241,10 +240,21 @@ # else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) # endif +#elif defined(__sparc__) +#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52) #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif +// Whether the addresses are sign-extended from the VMA range to the word. +// The SPARC64 Linux port implements this to split the VMA space into two +// non-contiguous halves with a huge hole in the middle. +#if defined(__sparc__) && SANITIZER_WORDSIZE == 64 +#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1 +#else +#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0 +#endif + // The AArch64 linux port uses the canonical syscall set as mandated by // the upstream linux community for all new ports. Other ports may still // use legacy syscalls. @@ -286,10 +296,10 @@ # define MSC_PREREQ(version) 0 #endif -#if defined(__arm64__) && SANITIZER_IOS -# define SANITIZER_NON_UNIQUE_TYPEINFO 1 -#else +#if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS) # define SANITIZER_NON_UNIQUE_TYPEINFO 0 +#else +# define SANITIZER_NON_UNIQUE_TYPEINFO 1 #endif // On linux, some architectures had an ABI transition from 64-bit long double diff --git a/lib/sanitizer_common/sanitizer_platform_interceptors.h b/lib/sanitizer_common/sanitizer_platform_interceptors.h index 4d146651b..675291d3c 100644 --- a/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_interceptors.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -143,6 +142,9 @@ #define SANITIZER_INTERCEPT_MEMMOVE 1 #define SANITIZER_INTERCEPT_MEMCPY 1 #define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_BCMP \ + SANITIZER_INTERCEPT_MEMCMP && \ + ((SI_POSIX && _GNU_SOURCE) || SI_NETBSD || SI_OPENBSD || SI_FREEBSD) #define SANITIZER_INTERCEPT_STRNDUP SI_POSIX #define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ @@ -281,6 +283,9 @@ #define SANITIZER_INTERCEPT_WCRTOMB \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ SI_SOLARIS) +#define SANITIZER_INTERCEPT_WCTOMB \ + (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ + SI_SOLARIS) #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_REALPATH SI_POSIX #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \ @@ -309,6 +314,7 @@ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX #define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX +#define SANITIZER_INTERCEPT_PTHREAD_SIGMASK SI_POSIX #define SANITIZER_INTERCEPT_BACKTRACE \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX @@ -357,6 +363,7 @@ #define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD #define SANITIZER_INTERCEPT_TMPNAM SI_POSIX #define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_TTYNAME SI_POSIX #define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX #define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX #define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS @@ -407,7 +414,8 @@ #else #define SANITIZER_INTERCEPT_AEABI_MEM 0 #endif -#define SANITIZER_INTERCEPT___BZERO SI_MAC +#define SANITIZER_INTERCEPT___BZERO SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_BZERO SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_FTIME \ (!SI_FREEBSD && !SI_NETBSD && !SI_OPENBSD && SI_POSIX) #define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS @@ -479,6 +487,7 @@ #define SANITIZER_INTERCEPT_CFREE \ (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \ SI_NOT_RTEMS) +#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_OPENBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID diff --git a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc index 377a62cab..034848742 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_freebsd.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h index 588bead2a..7e162a5e4 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_freebsd.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_linux.cc b/lib/sanitizer_common/sanitizer_platform_limits_linux.cc index 46e3b1813..14b0821cb 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_linux.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_linux.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc index b23b430d9..5860be228 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_netbsd.cc -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -122,7 +121,6 @@ #include <dev/ic/nvmeio.h> #include <dev/ir/irdaio.h> #include <dev/isa/isvio.h> -#include <dev/isa/satlinkio.h> #include <dev/isa/wtreg.h> #include <dev/iscsi/iscsi_ioctl.h> #include <dev/nvmm/nvmm_ioctl.h> @@ -639,7 +637,6 @@ unsigned struct_rf_recon_req_sz = sizeof(rf_recon_req); unsigned struct_rio_conf_sz = sizeof(rio_conf); unsigned struct_rio_interface_sz = sizeof(rio_interface); unsigned struct_rio_stats_sz = sizeof(rio_stats); -unsigned struct_satlink_id_sz = sizeof(satlink_id); unsigned struct_scan_io_sz = sizeof(scan_io); unsigned struct_scbusaccel_args_sz = sizeof(scbusaccel_args); unsigned struct_scbusiodetach_args_sz = sizeof(scbusiodetach_args); @@ -1105,9 +1102,6 @@ unsigned IOCTL_IRDA_GET_TURNAROUNDMASK = IRDA_GET_TURNAROUNDMASK; unsigned IOCTL_IRFRAMETTY_GET_DEVICE = IRFRAMETTY_GET_DEVICE; unsigned IOCTL_IRFRAMETTY_GET_DONGLE = IRFRAMETTY_GET_DONGLE; unsigned IOCTL_IRFRAMETTY_SET_DONGLE = IRFRAMETTY_SET_DONGLE; -unsigned IOCTL_SATIORESET = SATIORESET; -unsigned IOCTL_SATIOGID = SATIOGID; -unsigned IOCTL_SATIOSBUFSIZE = SATIOSBUFSIZE; unsigned IOCTL_ISV_CMD = ISV_CMD; unsigned IOCTL_WTQICMD = WTQICMD; unsigned IOCTL_ISCSI_GET_VERSION = ISCSI_GET_VERSION; diff --git a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h index 0c0c8a837..add9852ec 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_netbsd.h --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -803,7 +802,6 @@ extern unsigned struct_rf_recon_req_sz; extern unsigned struct_rio_conf_sz; extern unsigned struct_rio_interface_sz; extern unsigned struct_rio_stats_sz; -extern unsigned struct_satlink_id_sz; extern unsigned struct_scan_io_sz; extern unsigned struct_scbusaccel_args_sz; extern unsigned struct_scbusiodetach_args_sz; @@ -1266,9 +1264,6 @@ extern unsigned IOCTL_IRDA_GET_TURNAROUNDMASK; extern unsigned IOCTL_IRFRAMETTY_GET_DEVICE; extern unsigned IOCTL_IRFRAMETTY_GET_DONGLE; extern unsigned IOCTL_IRFRAMETTY_SET_DONGLE; -extern unsigned IOCTL_SATIORESET; -extern unsigned IOCTL_SATIOGID; -extern unsigned IOCTL_SATIOSBUFSIZE; extern unsigned IOCTL_ISV_CMD; extern unsigned IOCTL_WTQICMD; extern unsigned IOCTL_ISCSI_GET_VERSION; diff --git a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc index 7ab0ff60a..5a1b07fa9 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_openbsd.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h index d9899913e..6d8b06271 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_openbsd.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index ecc69bcea..b7fa6e8f7 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_posix.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h index de69852d3..f1a4fd7d3 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_posix.h ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc b/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc index 15b80a81a..3503eb2ea 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_solaris.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h index 08bd24cd2..eb5c5855b 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_solaris.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc index b965fb0da..91e7f0fb0 100644 --- a/lib/sanitizer_common/sanitizer_posix.cc +++ b/lib/sanitizer_common/sanitizer_posix.cc @@ -1,9 +1,8 @@ //===-- sanitizer_posix.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,9 +43,8 @@ uptr GetMmapGranularity() { void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); - uptr res = internal_mmap(nullptr, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, mem_type); int reserrno; if (UNLIKELY(internal_iserror(res, &reserrno))) ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report); @@ -67,9 +65,8 @@ void UnmapOrDie(void *addr, uptr size) { void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { size = RoundUpTo(size, GetPageSizeCached()); - uptr res = internal_mmap(nullptr, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, mem_type); int reserrno; if (UNLIKELY(internal_iserror(res, &reserrno))) { if (reserrno == ENOMEM) @@ -104,12 +101,9 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, } void *MmapNoReserveOrDie(uptr size, const char *mem_type) { - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap(nullptr, - RoundUpTo(size, PageSize), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, - -1, 0); + size = RoundUpTo(size, GetPageSizeCached()); + uptr p = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, mem_type); int reserrno; if (UNLIKELY(internal_iserror(p, &reserrno))) ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno); @@ -117,13 +111,12 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { return (void *)p; } -void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) { - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap((void*)(fixed_addr & ~(PageSize - 1)), - RoundUpTo(size, PageSize), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_FIXED, - -1, 0); +static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem, + const char *name) { + size = RoundUpTo(size, GetPageSizeCached()); + fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached()); + uptr p = MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, name); int reserrno; if (UNLIKELY(internal_iserror(p, &reserrno))) { if (tolerate_enomem && reserrno == ENOMEM) @@ -137,12 +130,12 @@ void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) { return (void *)p; } -void *MmapFixedOrDie(uptr fixed_addr, uptr size) { - return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/); +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/, name); } -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { - return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/); +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/, name); } bool MprotectNoAccess(uptr addr, uptr size) { @@ -344,6 +337,53 @@ bool ShouldMockFailureToOpen(const char *path) { internal_strncmp(path, "/proc/", 6) == 0; } +#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO +int GetNamedMappingFd(const char *name, uptr size, int *flags) { + if (!common_flags()->decorate_proc_maps || !name) + return -1; + char shmname[200]; + CHECK(internal_strlen(name) < sizeof(shmname) - 10); + internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu [%s]", + internal_getpid(), name); + int fd = ReserveStandardFds( + internal_open(shmname, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, S_IRWXU)); + CHECK_GE(fd, 0); + int res = internal_ftruncate(fd, size); + CHECK_EQ(0, res); + res = internal_unlink(shmname); + CHECK_EQ(0, res); + *flags &= ~(MAP_ANON | MAP_ANONYMOUS); + return fd; +} +#else +int GetNamedMappingFd(const char *name, uptr size, int *flags) { + return -1; +} +#endif + +#if SANITIZER_ANDROID +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 +void DecorateMapping(uptr addr, uptr size, const char *name) { + if (!common_flags()->decorate_proc_maps || !name) + return; + CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, addr, size, + (uptr)name) == 0); +} +#else +void DecorateMapping(uptr addr, uptr size, const char *name) { +} +#endif + +uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name) { + int fd = GetNamedMappingFd(name, length, &flags); + uptr res = internal_mmap(addr, length, prot, flags, fd, 0); + if (!internal_iserror(res)) + DecorateMapping(res, length, name); + return res; +} + + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/lib/sanitizer_common/sanitizer_posix.h b/lib/sanitizer_common/sanitizer_posix.h index 04a76445e..7b88e8815 100644 --- a/lib/sanitizer_common/sanitizer_posix.h +++ b/lib/sanitizer_common/sanitizer_posix.h @@ -1,9 +1,8 @@ //===-- sanitizer_posix.h -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -105,6 +104,18 @@ fd_t ReserveStandardFds(fd_t fd); bool ShouldMockFailureToOpen(const char *path); +// Create a non-file mapping with a given /proc/self/maps name. +uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name); + +// Platforms should implement at most one of these. +// 1. Provide a pre-decorated file descriptor to use instead of an anonymous +// mapping. +int GetNamedMappingFd(const char *name, uptr size, int *flags); +// 2. Add name to an existing anonymous mapping. The caller must keep *name +// alive at least as long as the mapping exists. +void DecorateMapping(uptr addr, uptr size, const char *name); + + } // namespace __sanitizer #endif // SANITIZER_POSIX_H diff --git a/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 3006e60d8..efe51ec20 100644 --- a/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_posix_libcdep.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -117,10 +116,6 @@ bool StackSizeIsUnlimited() { return (stack_size == RLIM_INFINITY); } -uptr GetStackSizeLimitInBytes() { - return (uptr)getlim(RLIMIT_STACK); -} - void SetStackSizeLimitInBytes(uptr limit) { setlim(RLIMIT_STACK, (rlim_t)limit); CHECK(!StackSizeIsUnlimited()); @@ -308,37 +303,11 @@ void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { MemoryMappingLayout::CacheMemoryMappings(); } -#if SANITIZER_ANDROID || SANITIZER_GO -int GetNamedMappingFd(const char *name, uptr size) { - return -1; -} -#else -int GetNamedMappingFd(const char *name, uptr size) { - if (!common_flags()->decorate_proc_maps) - return -1; - char shmname[200]; - CHECK(internal_strlen(name) < sizeof(shmname) - 10); - internal_snprintf(shmname, sizeof(shmname), "%zu [%s]", internal_getpid(), - name); - int fd = shm_open(shmname, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); - CHECK_GE(fd, 0); - int res = internal_ftruncate(fd, size); - CHECK_EQ(0, res); - res = shm_unlink(shmname); - CHECK_EQ(0, res); - return fd; -} -#endif - bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { - int fd = name ? GetNamedMappingFd(name, size) : -1; - unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; - if (fd == -1) flags |= MAP_ANON; - - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap((void *)(fixed_addr & ~(PageSize - 1)), - RoundUpTo(size, PageSize), PROT_READ | PROT_WRITE, - flags, fd, 0); + size = RoundUpTo(size, GetPageSizeCached()); + fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached()); + uptr p = MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANON, name); int reserrno; if (internal_iserror(p, &reserrno)) { Report("ERROR: %s failed to " @@ -351,12 +320,8 @@ bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { } uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { - // We don't pass `name` along because, when you enable `decorate_proc_maps` - // AND actually use a named mapping AND are using a sanitizer intercepting - // `open` (e.g. TSAN, ESAN), then you'll get a failure during initialization. - // TODO(flowerhack): Fix the implementation of GetNamedMappingFd to solve - // this problem. - base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size) : MmapNoAccess(size); + base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size, name) + : MmapNoAccess(size); size_ = size; name_ = name; (void)os_handle_; // unsupported @@ -365,12 +330,14 @@ uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { // Uses fixed_addr for now. // Will use offset instead once we've implemented this function for real. -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { - return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) { + return reinterpret_cast<uptr>( + MmapFixedOrDieOnFatalError(fixed_addr, size, name)); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { - return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size, + const char *name) { + return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size, name)); } void ReservedAddressRange::Unmap(uptr addr, uptr size) { @@ -385,12 +352,9 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) { } void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { - int fd = name ? GetNamedMappingFd(name, size) : -1; - unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; - if (fd == -1) flags |= MAP_ANON; - - return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd, - 0); + return (void *)MmapNamed((void *)fixed_addr, size, PROT_NONE, + MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANON, + name); } void *MmapNoAccess(uptr size) { diff --git a/lib/sanitizer_common/sanitizer_printf.cc b/lib/sanitizer_common/sanitizer_printf.cc index 4315f6516..f4e17a8f5 100644 --- a/lib/sanitizer_common/sanitizer_printf.cc +++ b/lib/sanitizer_common/sanitizer_printf.cc @@ -1,9 +1,8 @@ //===-- sanitizer_printf.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps.h b/lib/sanitizer_common/sanitizer_procmaps.h index acb7104f3..052027111 100644 --- a/lib/sanitizer_common/sanitizer_procmaps.h +++ b/lib/sanitizer_common/sanitizer_procmaps.h @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_bsd.cc b/lib/sanitizer_common/sanitizer_procmaps_bsd.cc index 362a424d7..c38bafd9f 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_bsd.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_bsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_bsd.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_common.cc b/lib/sanitizer_common/sanitizer_procmaps_common.cc index 17d61b6c5..4a2b0a047 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_common.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/lib/sanitizer_common/sanitizer_procmaps_linux.cc index cf9cb25ba..fd5e03619 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_linux.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 267c960b5..148910f42 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_mac.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc index 49bb46c31..6e4c59c10 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -1,7 +1,8 @@ //===-- sanitizer_procmaps_solaris.cc -------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_quarantine.h b/lib/sanitizer_common/sanitizer_quarantine.h index d4aa12fbe..992f23152 100644 --- a/lib/sanitizer_common/sanitizer_quarantine.h +++ b/lib/sanitizer_common/sanitizer_quarantine.h @@ -1,9 +1,8 @@ //===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_report_decorator.h b/lib/sanitizer_common/sanitizer_report_decorator.h index a7878684a..d276c2cdd 100644 --- a/lib/sanitizer_common/sanitizer_report_decorator.h +++ b/lib/sanitizer_common/sanitizer_report_decorator.h @@ -1,9 +1,8 @@ //===-- sanitizer_report_decorator.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_ring_buffer.h b/lib/sanitizer_common/sanitizer_ring_buffer.h index d15f27fd4..d60014d8f 100644 --- a/lib/sanitizer_common/sanitizer_ring_buffer.h +++ b/lib/sanitizer_common/sanitizer_ring_buffer.h @@ -1,9 +1,8 @@ //===-- sanitizer_ring_buffer.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_rtems.cc b/lib/sanitizer_common/sanitizer_rtems.cc index 76aebe419..ffc21b958 100644 --- a/lib/sanitizer_common/sanitizer_rtems.cc +++ b/lib/sanitizer_common/sanitizer_rtems.cc @@ -1,9 +1,8 @@ //===-- sanitizer_rtems.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_rtems.h b/lib/sanitizer_common/sanitizer_rtems.h index 968fa66e1..e8adfd500 100644 --- a/lib/sanitizer_common/sanitizer_rtems.h +++ b/lib/sanitizer_common/sanitizer_rtems.h @@ -1,9 +1,8 @@ //===-- sanitizer_rtems.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/lib/sanitizer_common/sanitizer_signal_interceptors.inc index f51fc049e..68d9eb659 100644 --- a/lib/sanitizer_common/sanitizer_signal_interceptors.inc +++ b/lib/sanitizer_common/sanitizer_signal_interceptors.inc @@ -1,9 +1,8 @@ //===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_solaris.cc b/lib/sanitizer_common/sanitizer_solaris.cc index cc0201c7a..12d03fab8 100644 --- a/lib/sanitizer_common/sanitizer_solaris.cc +++ b/lib/sanitizer_common/sanitizer_solaris.cc @@ -1,7 +1,8 @@ //===-- sanitizer_solaris.cc ----------------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -124,6 +125,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +DECLARE__REAL_AND_INTERNAL(uptr, dup, int oldfd) { + return _REAL(dup)(oldfd); +} + DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) { return _REAL(dup2)(oldfd, newfd); } diff --git a/lib/sanitizer_common/sanitizer_stackdepot.cc b/lib/sanitizer_common/sanitizer_stackdepot.cc index 6aab98485..1cdedfa32 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepot.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "sanitizer_stackdepot.h" #include "sanitizer_common.h" +#include "sanitizer_hash.h" #include "sanitizer_stackdepotbase.h" namespace __sanitizer { @@ -50,23 +50,9 @@ struct StackDepotNode { return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr); } static u32 hash(const args_type &args) { - // murmur2 - const u32 m = 0x5bd1e995; - const u32 seed = 0x9747b28c; - const u32 r = 24; - u32 h = seed ^ (args.size * sizeof(uptr)); - for (uptr i = 0; i < args.size; i++) { - u32 k = args.trace[i]; - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - } - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h; + MurMur2HashBuilder H(args.size * sizeof(uptr)); + for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]); + return H.get(); } static bool is_valid(const args_type &args) { return args.size > 0 && args.trace; diff --git a/lib/sanitizer_common/sanitizer_stackdepot.h b/lib/sanitizer_common/sanitizer_stackdepot.h index e22ed2e38..bf29cb9a0 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/lib/sanitizer_common/sanitizer_stackdepot.h @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepot.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stackdepotbase.h b/lib/sanitizer_common/sanitizer_stackdepotbase.h index 4ec77b467..ef1b4f7f7 100644 --- a/lib/sanitizer_common/sanitizer_stackdepotbase.h +++ b/lib/sanitizer_common/sanitizer_stackdepotbase.h @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepotbase.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace.cc b/lib/sanitizer_common/sanitizer_stacktrace.cc index 21976b6b1..26474037f 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,10 +17,9 @@ namespace __sanitizer { uptr StackTrace::GetNextInstructionPc(uptr pc) { -#if defined(__mips__) +#if defined(__sparc__) || defined(__mips__) return pc + 8; -#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ - defined(__aarch64__) +#elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__) return pc + 4; #else return pc + 1; @@ -50,6 +48,7 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) { static inline uhwptr *GetCanonicFrame(uptr bp, uptr stack_top, uptr stack_bottom) { + CHECK_GT(stack_top, stack_bottom); #ifdef __arm__ if (!IsValidFrame(bp, stack_top, stack_bottom)) return 0; uhwptr *bp_prev = (uhwptr *)bp; @@ -68,10 +67,11 @@ static inline uhwptr *GetCanonicFrame(uptr bp, #endif } -void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, - uptr stack_bottom, u32 max_depth) { - const uptr kPageSize = GetPageSizeCached(); +void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, + uptr stack_bottom, u32 max_depth) { + // TODO(yln): add arg sanity check for stack_top/stack_bottom CHECK_GE(max_depth, 2); + const uptr kPageSize = GetPageSizeCached(); trace_buffer[0] = pc; size = 1; if (stack_top < 4096) return; // Sanity check for stack top. diff --git a/lib/sanitizer_common/sanitizer_stacktrace.h b/lib/sanitizer_common/sanitizer_stacktrace.h index 450a40a90..f1f29e9f3 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/lib/sanitizer_common/sanitizer_stacktrace.h @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,9 +16,11 @@ namespace __sanitizer { +struct BufferedStackTrace; + static const u32 kStackTraceMax = 256; -#if defined(__sparc__) || (SANITIZER_LINUX && defined(__mips__)) +#if SANITIZER_LINUX && defined(__mips__) # define SANITIZER_CAN_FAST_UNWIND 0 #elif SANITIZER_WINDOWS # define SANITIZER_CAN_FAST_UNWIND 0 @@ -59,7 +60,7 @@ struct StackTrace { static bool WillUseFastUnwind(bool request_fast_unwind) { if (!SANITIZER_CAN_FAST_UNWIND) return false; - else if (!SANITIZER_CAN_SLOW_UNWIND) + if (!SANITIZER_CAN_SLOW_UNWIND) return true; return request_fast_unwind; } @@ -97,6 +98,23 @@ struct BufferedStackTrace : public StackTrace { BufferedStackTrace() : StackTrace(trace_buffer, 0), top_frame_bp(0) {} void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0); + + // Get the stack trace with the given pc and bp. + // The pc will be in the position 0 of the resulting stack trace. + // The bp may refer to the current frame or to the caller's frame. + void Unwind(uptr pc, uptr bp, void *context, bool request_fast, + u32 max_depth = kStackTraceMax) { + top_frame_bp = (max_depth > 0) ? bp : 0; + // Small max_depth optimization + if (max_depth <= 1) { + if (max_depth == 1) + trace_buffer[0] = pc; + size = max_depth; + return; + } + UnwindImpl(pc, bp, context, request_fast, max_depth); + } + void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind); @@ -106,16 +124,23 @@ struct BufferedStackTrace : public StackTrace { } private: - void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, - u32 max_depth); - void SlowUnwindStack(uptr pc, u32 max_depth); - void SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth); + // Every runtime defines its own implementation of this method + void UnwindImpl(uptr pc, uptr bp, void *context, bool request_fast, + u32 max_depth); + + // UnwindFast/Slow have platform-specific implementations + void UnwindFast(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, + u32 max_depth); + void UnwindSlow(uptr pc, u32 max_depth); + void UnwindSlow(uptr pc, void *context, u32 max_depth); + void PopStackFrames(uptr count); uptr LocatePcInTrace(uptr pc); BufferedStackTrace(const BufferedStackTrace &) = delete; void operator=(const BufferedStackTrace &) = delete; + + friend class FastUnwindTest; }; // Check if given pointer points into allocated stack area. @@ -127,21 +152,23 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) { // Use this macro if you want to print stack trace with the caller // of the current function in the top frame. -#define GET_CALLER_PC_BP_SP \ - uptr bp = GET_CURRENT_FRAME(); \ - uptr pc = GET_CALLER_PC(); \ - uptr local_stack; \ - uptr sp = (uptr)&local_stack - #define GET_CALLER_PC_BP \ uptr bp = GET_CURRENT_FRAME(); \ uptr pc = GET_CALLER_PC(); +#define GET_CALLER_PC_BP_SP \ + GET_CALLER_PC_BP; \ + uptr local_stack; \ + uptr sp = (uptr)&local_stack + // Use this macro if you want to print stack trace with the current // function in the top frame. -#define GET_CURRENT_PC_BP_SP \ +#define GET_CURRENT_PC_BP \ uptr bp = GET_CURRENT_FRAME(); \ - uptr pc = StackTrace::GetCurrentPc(); \ + uptr pc = StackTrace::GetCurrentPc() + +#define GET_CURRENT_PC_BP_SP \ + GET_CURRENT_PC_BP; \ uptr local_stack; \ uptr sp = (uptr)&local_stack diff --git a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index c87b18e1b..859032ba8 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_libcdep.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -58,6 +57,8 @@ void StackTrace::Print() const { void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind) { + // Ensures all call sites get what they requested. + CHECK_EQ(request_fast_unwind, WillUseFastUnwind(request_fast_unwind)); top_frame_bp = (max_depth > 0) ? bp : 0; // Avoid doing any work for small max_depth. if (max_depth == 0) { @@ -72,14 +73,14 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, if (!WillUseFastUnwind(request_fast_unwind)) { #if SANITIZER_CAN_SLOW_UNWIND if (context) - SlowUnwindStackWithContext(pc, context, max_depth); + UnwindSlow(pc, context, max_depth); else - SlowUnwindStack(pc, max_depth); + UnwindSlow(pc, max_depth); #else UNREACHABLE("slow unwind requested but not available"); #endif } else { - FastUnwindStack(pc, bp, stack_top, stack_bottom, max_depth); + UnwindFast(pc, bp, stack_top, stack_bottom, max_depth); } } diff --git a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index f2b337433..17bbf6c0b 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/lib/sanitizer_common/sanitizer_stacktrace_printer.h index ce85bd7f2..f7f7629f7 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_printer.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc b/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc index f41a3cefb..b238cfbc2 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_sparc.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,47 +12,74 @@ // Implemention of fast stack unwinding for Sparc. //===----------------------------------------------------------------------===// -// This file is ported to Sparc v8, but it should be easy to port to -// Sparc v9. -#if defined(__sparcv8__) || defined(__sparcv8) || defined(__sparc_v8__) +#if defined(__sparc__) + +#if defined(__arch64__) || defined(__sparcv9) +#define STACK_BIAS 2047 +#else +#define STACK_BIAS 0 +#endif #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" namespace __sanitizer { -void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, - uptr stack_bottom, u32 max_depth) { - const uptr kPageSize = GetPageSizeCached(); +void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, + uptr stack_bottom, u32 max_depth) { + // TODO(yln): add arg sanity check for stack_top/stack_bottom CHECK_GE(max_depth, 2); + const uptr kPageSize = GetPageSizeCached(); +#if defined(__GNUC__) + // __builtin_return_address returns the address of the call instruction + // on the SPARC and not the return address, so we need to compensate. + trace_buffer[0] = GetNextInstructionPc(pc); +#else trace_buffer[0] = pc; +#endif size = 1; if (stack_top < 4096) return; // Sanity check for stack top. // Flush register windows to memory +#if defined(__sparc_v9__) || defined(__sparcv9__) || defined(__sparcv9) + asm volatile("flushw" ::: "memory"); +#else asm volatile("ta 3" ::: "memory"); - uhwptr *frame = (uhwptr*)bp; +#endif + // On the SPARC, the return address is not in the frame, it is in a + // register. There is no way to access it off of the current frame + // pointer, but it can be accessed off the previous frame pointer by + // reading the value from the register window save area. + uptr prev_bp = GET_CURRENT_FRAME(); + uptr next_bp = prev_bp; + unsigned int i = 0; + while (next_bp != bp && IsAligned(next_bp, sizeof(uhwptr)) && i++ < 8) { + prev_bp = next_bp; + next_bp = (uptr)((uhwptr *)next_bp)[14] + STACK_BIAS; + } + if (next_bp == bp) + bp = prev_bp; // Lowest possible address that makes sense as the next frame pointer. // Goes up as we walk the stack. uptr bottom = stack_bottom; // Avoid infinite loop when frame == frame[0] by using frame > prev_frame. - while (IsValidFrame((uptr)frame, stack_top, bottom) && - IsAligned((uptr)frame, sizeof(*frame)) && + while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) && size < max_depth) { - uhwptr pc1 = frame[15]; + uhwptr pc1 = ((uhwptr *)bp)[15]; // Let's assume that any pointer in the 0th page is invalid and // stop unwinding here. If we're adding support for a platform // where this isn't true, we need to reconsider this check. if (pc1 < kPageSize) break; if (pc1 != pc) { - trace_buffer[size++] = (uptr) pc1; + // %o7 contains the address of the call instruction and not the + // return address, so we need to compensate. + trace_buffer[size++] = GetNextInstructionPc((uptr)pc1); } - bottom = (uptr)frame; - frame = (uhwptr*)frame[14]; + bottom = bp; + bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS; } } } // namespace __sanitizer -#endif // !defined(__sparcv8__) && !defined(__sparcv8) && - // !defined(__sparc_v8__) +#endif // !defined(__sparc__) diff --git a/lib/sanitizer_common/sanitizer_stoptheworld.h b/lib/sanitizer_common/sanitizer_stoptheworld.h index 20b49ae78..4e4240057 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld.h +++ b/lib/sanitizer_common/sanitizer_stoptheworld.h @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index fe4fe86da..716f0d226 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_linux_libcdep.cc ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc b/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc index d84ebef3b..e79edc40f 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc +++ b/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_mac.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_suppressions.cc b/lib/sanitizer_common/sanitizer_suppressions.cc index 232171ed5..12ecd9a2e 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/lib/sanitizer_common/sanitizer_suppressions.cc @@ -1,9 +1,8 @@ //===-- sanitizer_suppressions.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,6 +30,7 @@ SuppressionContext::SuppressionContext(const char *suppression_types[], internal_memset(has_suppression_type_, 0, suppression_types_num_); } +#if !SANITIZER_FUCHSIA static bool GetPathAssumingFileIsRelativeToExec(const char *file_path, /*out*/char *new_file_path, uptr new_file_path_size) { @@ -47,20 +47,30 @@ static bool GetPathAssumingFileIsRelativeToExec(const char *file_path, return false; } +static const char *FindFile(const char *file_path, + /*out*/char *new_file_path, + uptr new_file_path_size) { + // If we cannot find the file, check if its location is relative to + // the location of the executable. + if (!FileExists(file_path) && !IsAbsolutePath(file_path) && + GetPathAssumingFileIsRelativeToExec(file_path, new_file_path, + new_file_path_size)) { + return new_file_path; + } + return file_path; +} +#else +static const char *FindFile(const char *file_path, char *, uptr) { + return file_path; +} +#endif + void SuppressionContext::ParseFromFile(const char *filename) { if (filename[0] == '\0') return; -#if !SANITIZER_FUCHSIA - // If we cannot find the file, check if its location is relative to - // the location of the executable. InternalScopedString new_file_path(kMaxPathLength); - if (!FileExists(filename) && !IsAbsolutePath(filename) && - GetPathAssumingFileIsRelativeToExec(filename, new_file_path.data(), - new_file_path.size())) { - filename = new_file_path.data(); - } -#endif // !SANITIZER_FUCHSIA + filename = FindFile(filename, new_file_path.data(), new_file_path.size()); // Read the file. VPrintf(1, "%s: reading suppressions file at %s\n", @@ -94,7 +104,7 @@ bool SuppressionContext::Match(const char *str, const char *type, } static const char *StripPrefix(const char *str, const char *prefix) { - while (str && *str == *prefix) { + while (*str && *str == *prefix) { str++; prefix++; } diff --git a/lib/sanitizer_common/sanitizer_suppressions.h b/lib/sanitizer_common/sanitizer_suppressions.h index 0ca875a2d..f9da7af7e 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.h +++ b/lib/sanitizer_common/sanitizer_suppressions.h @@ -1,9 +1,8 @@ //===-- sanitizer_suppressions.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer.cc b/lib/sanitizer_common/sanitizer_symbolizer.cc index 672c93668..b27a49d4f 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer.h b/lib/sanitizer_common/sanitizer_symbolizer.h index e08eb0d66..8c318b8da 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/lib/sanitizer_common/sanitizer_symbolizer.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h b/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h index b241b9dbc..c4061e38c 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_fuchsia.h -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/lib/sanitizer_common/sanitizer_symbolizer_internal.h index a2ac11882..a4a4f1106 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_internal.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc index e98fab01d..8a20e062c 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libbacktrace.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h index ddfd47559..e2a0f7142 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libbacktrace.h ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc index f2ee7baad..1c2ff6dcb 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libcdep.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc index f08cb9f97..d35665719 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_mac.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_mac.h b/lib/sanitizer_common/sanitizer_symbolizer_mac.h index 068644de3..68521375e 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_mac.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_mac.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_mac.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_markup.cc b/lib/sanitizer_common/sanitizer_symbolizer_markup.cc index c62dc90fa..aee49b4c4 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_markup.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_markup.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_markup.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -118,7 +117,7 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { : _URC_NO_REASON); } -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); size = 0; UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; @@ -133,9 +132,9 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { trace_buffer[0] = pc; } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { - CHECK_NE(context, nullptr); +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); + CHECK_GE(max_depth, 2); UNREACHABLE("signal context doesn't exist"); } #endif // SANITIZER_CAN_SLOW_UNWIND diff --git a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index d74a6f44e..2df3a90ba 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_posix_libcdep.cc -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_report.cc b/lib/sanitizer_common/sanitizer_symbolizer_report.cc index fd26d4cfa..f4167d160 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_report.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_report.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_report.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -104,9 +103,11 @@ void ReportMmapWriteExec(int prot) { GET_CALLER_PC_BP_SP; (void)sp; bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) + if (StackTrace::WillUseFastUnwind(fast)) { GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, fast); + stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, true); + } else + stack->Unwind(kStackTraceMax, pc, 0, nullptr, 0, 0, false); Printf("%s", d.Warning()); Report("WARNING: %s: writable-executable page usage\n", SanitizerToolName); diff --git a/lib/sanitizer_common/sanitizer_symbolizer_rtems.h b/lib/sanitizer_common/sanitizer_symbolizer_rtems.h index 62356ef6e..3371092e0 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_rtems.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_rtems.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_rtems.h -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_win.cc b/lib/sanitizer_common/sanitizer_symbolizer_win.cc index 6ec75831a..d75fe9ae1 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_win.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_win.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_generic.inc b/lib/sanitizer_common/sanitizer_syscall_generic.inc index ddfb26397..a43ce3efa 100644 --- a/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_generic.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 7ab1d7641..56c5e9922 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_aarch64.inc --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc index b4fd0962a..121a9445b 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 9853a6a67..67e8686d1 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_x86_64.inc ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc b/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc index 75aea2760..21b521669 100644 --- a/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc +++ b/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscalls_netbsd.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_termination.cc b/lib/sanitizer_common/sanitizer_termination.cc index 35e4403ad..a011cce42 100644 --- a/lib/sanitizer_common/sanitizer_termination.cc +++ b/lib/sanitizer_common/sanitizer_termination.cc @@ -1,9 +1,8 @@ //===-- sanitizer_termination.cc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/sanitizer_common/sanitizer_thread_registry.cc b/lib/sanitizer_common/sanitizer_thread_registry.cc index eb35cb6c5..02691287d 100644 --- a/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -1,9 +1,8 @@ //===-- sanitizer_thread_registry.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ namespace __sanitizer { ThreadContextBase::ThreadContextBase(u32 tid) : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0), - status(ThreadStatusInvalid), - detached(false), workerthread(false), parent_tid(0), next(0) { + status(ThreadStatusInvalid), detached(false), + thread_type(ThreadType::Regular), parent_tid(0), next(0) { name[0] = '\0'; atomic_store(&thread_destroyed, 0, memory_order_release); } @@ -71,11 +70,11 @@ void ThreadContextBase::SetFinished() { OnFinished(); } -void ThreadContextBase::SetStarted(tid_t _os_id, bool _workerthread, +void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg) { status = ThreadStatusRunning; os_id = _os_id; - workerthread = _workerthread; + thread_type = _thread_type; OnStarted(arg); } @@ -303,7 +302,7 @@ void ThreadRegistry::FinishThread(u32 tid) { tctx->SetDestroyed(); } -void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, +void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg) { BlockingMutexLock l(&mtx_); running_threads_++; @@ -311,7 +310,7 @@ void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_EQ(ThreadStatusCreated, tctx->status); - tctx->SetStarted(os_id, workerthread, arg); + tctx->SetStarted(os_id, thread_type, arg); } void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) { diff --git a/lib/sanitizer_common/sanitizer_thread_registry.h b/lib/sanitizer_common/sanitizer_thread_registry.h index 30dc603fe..493aa988f 100644 --- a/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/lib/sanitizer_common/sanitizer_thread_registry.h @@ -1,9 +1,8 @@ //===-- sanitizer_thread_registry.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,12 @@ enum ThreadStatus { ThreadStatusDead // Joined, but some info is still available. }; +enum class ThreadType { + Regular, // Normal thread + Worker, // macOS Grand Central Dispatch (GCD) worker thread + Fiber, // Fiber +}; + // Generic thread context. Specific sanitizer tools may inherit from it. // If thread is dead, context may optionally be reused for a new thread. class ThreadContextBase { @@ -45,7 +50,7 @@ class ThreadContextBase { ThreadStatus status; bool detached; - bool workerthread; + ThreadType thread_type; u32 parent_tid; ThreadContextBase *next; // For storing thread contexts in a list. @@ -57,7 +62,7 @@ class ThreadContextBase { void SetDead(); void SetJoined(void *arg); void SetFinished(); - void SetStarted(tid_t _os_id, bool _workerthread, void *arg); + void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg); void SetCreated(uptr _user_id, u64 _unique_id, bool _detached, u32 _parent_tid, void *arg); void Reset(); @@ -121,7 +126,7 @@ class ThreadRegistry { void DetachThread(u32 tid, void *arg); void JoinThread(u32 tid, void *arg); void FinishThread(u32 tid); - void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg); + void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg); void SetThreadUserId(u32 tid, uptr user_id); private: diff --git a/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/lib/sanitizer_common/sanitizer_tls_get_addr.cc index 85dc806df..5e33e2a51 100644 --- a/lib/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/lib/sanitizer_common/sanitizer_tls_get_addr.cc @@ -1,9 +1,8 @@ //===-- sanitizer_tls_get_addr.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_tls_get_addr.h b/lib/sanitizer_common/sanitizer_tls_get_addr.h index 199a3b2e9..cc178d3d1 100644 --- a/lib/sanitizer_common/sanitizer_tls_get_addr.h +++ b/lib/sanitizer_common/sanitizer_tls_get_addr.h @@ -1,9 +1,8 @@ //===-- sanitizer_tls_get_addr.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_type_traits.cc b/lib/sanitizer_common/sanitizer_type_traits.cc index 27fec6e1f..e3e431a13 100644 --- a/lib/sanitizer_common/sanitizer_type_traits.cc +++ b/lib/sanitizer_common/sanitizer_type_traits.cc @@ -1,9 +1,8 @@ //===-- sanitizer_type_traits.cc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_type_traits.h b/lib/sanitizer_common/sanitizer_type_traits.h index 4495f2c34..2a58d9874 100644 --- a/lib/sanitizer_common/sanitizer_type_traits.h +++ b/lib/sanitizer_common/sanitizer_type_traits.h @@ -1,9 +1,8 @@ //===-- sanitizer_type_traits.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,6 +38,25 @@ struct is_same : public false_type {}; template <typename T> struct is_same<T, T> : public true_type {}; +// conditional<B, T, F> +// +// Defines type as T if B is true or as F otherwise. +// E.g. the following is true +// +// ``` +// is_same<int, conditional<true, int, double>::type>::value +// is_same<double, conditional<false, int, double>::type>::value +// ``` +template <bool B, class T, class F> +struct conditional { + using type = T; +}; + +template <class T, class F> +struct conditional<false, T, F> { + using type = F; +}; + } // namespace __sanitizer #endif diff --git a/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index c7a5ec86f..30581e848 100644 --- a/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_unwind_linux_libcdep.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,7 @@ namespace __sanitizer { -//------------------------- SlowUnwindStack ----------------------------------- +//---------------------------- UnwindSlow -------------------------------------- typedef struct { uptr absolute_pc; @@ -120,7 +119,7 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { return UNWIND_CONTINUE; } -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); size = 0; UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; @@ -136,14 +135,20 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { if (to_pop == 0 && size > 1) to_pop = 1; PopStackFrames(to_pop); +#if defined(__GNUC__) && defined(__sparc__) + // __builtin_return_address returns the address of the call instruction + // on the SPARC and not the return address, so we need to compensate. + trace_buffer[0] = GetNextInstructionPc(pc); +#else trace_buffer[0] = pc; +#endif } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); CHECK_GE(max_depth, 2); if (!unwind_backtrace_signal_arch) { - SlowUnwindStack(pc, max_depth); + UnwindSlow(pc, max_depth); return; } diff --git a/lib/sanitizer_common/sanitizer_unwind_win.cc b/lib/sanitizer_common/sanitizer_unwind_win.cc index 62bac4e9b..93908abab 100644 --- a/lib/sanitizer_common/sanitizer_unwind_win.cc +++ b/lib/sanitizer_common/sanitizer_unwind_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_unwind_win.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ using namespace __sanitizer; #if !SANITIZER_GO -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); // FIXME: CaptureStackBackTrace might be too slow for us. // FIXME: Compare with StackWalk64. @@ -40,8 +39,9 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { PopStackFrames(pc_location); } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); + CHECK_GE(max_depth, 2); CONTEXT ctx = *(CONTEXT *)context; STACKFRAME64 stack_frame; memset(&stack_frame, 0, sizeof(stack_frame)); diff --git a/lib/sanitizer_common/sanitizer_vector.h b/lib/sanitizer_common/sanitizer_vector.h index 0632ccce2..4b9ae7db4 100644 --- a/lib/sanitizer_common/sanitizer_vector.h +++ b/lib/sanitizer_common/sanitizer_vector.h @@ -1,9 +1,8 @@ //===-- sanitizer_vector.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc index 9a574dd23..457cecb8c 100644 --- a/lib/sanitizer_common/sanitizer_win.cc +++ b/lib/sanitizer_common/sanitizer_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,18 @@ #if defined(PSAPI_VERSION) && PSAPI_VERSION == 1 #pragma comment(lib, "psapi") #endif +#if SANITIZER_WIN_TRACE +#include <traceloggingprovider.h> +// Windows trace logging provider init +#pragma comment(lib, "advapi32.lib") +TRACELOGGING_DECLARE_PROVIDER(g_asan_provider); +// GUID must be the same in utils/AddressSanitizerLoggingProvider.wprp +TRACELOGGING_DEFINE_PROVIDER(g_asan_provider, "AddressSanitizerLoggingProvider", + (0x6c6c766d, 0x3846, 0x4e6a, 0xa4, 0xfb, 0x5b, + 0x53, 0x0b, 0xd0, 0xf3, 0xfa)); +#else +#define TraceLoggingUnregister(x) +#endif // A macro to tell the compiler that this part of the code cannot be reached, // if the compiler supports this feature. Since we're using this in @@ -230,7 +241,7 @@ bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { // Memory space mapped by 'MmapFixedOrDie' must have been reserved by // 'MmapFixedNoAccess'. -void *MmapFixedOrDie(uptr fixed_addr, uptr size) { +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); if (p == 0) { @@ -244,11 +255,12 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) { // Uses fixed_addr for now. // Will use offset instead once we've implemented this function for real. -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) { return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size, + const char *name) { return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); } @@ -261,7 +273,7 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) { UnmapOrDie(reinterpret_cast<void*>(addr), size); } -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); if (p == 0) { @@ -487,8 +499,14 @@ bool IsPathSeparator(const char c) { return c == '\\' || c == '/'; } +static bool IsAlpha(char c) { + c = ToLower(c); + return c >= 'a' && c <= 'z'; +} + bool IsAbsolutePath(const char *path) { - UNIMPLEMENTED(); + return path != nullptr && IsAlpha(path[0]) && path[1] == ':' && + IsPathSeparator(path[2]); } void SleepForSeconds(int seconds) { @@ -646,6 +664,7 @@ int Atexit(void (*function)(void)) { } static int RunAtexit() { + TraceLoggingUnregister(g_asan_provider); int ret = 0; for (uptr i = 0; i < atexit_functions.size(); ++i) { ret |= atexit(atexit_functions[i]); @@ -743,6 +762,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { + TraceLoggingUnregister(g_asan_provider); // ExitProcess runs some finalizers, so use TerminateProcess to avoid that. // The debugger doesn't stop on TerminateProcess like it does on ExitProcess, // so add our own breakpoint here. @@ -1064,6 +1084,32 @@ u32 GetNumberOfCPUs() { return sysinfo.dwNumberOfProcessors; } +#if SANITIZER_WIN_TRACE +// TODO(mcgov): Rename this project-wide to PlatformLogInit +void AndroidLogInit(void) { + HRESULT hr = TraceLoggingRegister(g_asan_provider); + if (!SUCCEEDED(hr)) + return; +} + +void SetAbortMessage(const char *) {} + +void LogFullErrorReport(const char *buffer) { + if (common_flags()->log_to_syslog) { + InternalMmapVector<wchar_t> filename; + DWORD filename_length = 0; + do { + filename.resize(filename.size() + 0x100); + filename_length = + GetModuleFileNameW(NULL, filename.begin(), filename.size()); + } while (filename_length >= filename.size()); + TraceLoggingWrite(g_asan_provider, "AsanReportEvent", + TraceLoggingValue(filename.begin(), "ExecutableName"), + TraceLoggingValue(buffer, "AsanReportContents")); + } +} +#endif // SANITIZER_WIN_TRACE + } // namespace __sanitizer #endif // _WIN32 diff --git a/lib/sanitizer_common/sanitizer_win.h b/lib/sanitizer_common/sanitizer_win.h index 23e01ab75..ff8939ca5 100644 --- a/lib/sanitizer_common/sanitizer_win.h +++ b/lib/sanitizer_common/sanitizer_win.h @@ -1,9 +1,8 @@ //===-- sanitizer_win.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win_defs.h b/lib/sanitizer_common/sanitizer_win_defs.h index 10fc2d021..bcd94a08d 100644 --- a/lib/sanitizer_common/sanitizer_win_defs.h +++ b/lib/sanitizer_common/sanitizer_win_defs.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_defs.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win_dll_thunk.cc b/lib/sanitizer_common/sanitizer_win_dll_thunk.cc index 4fb4650be..5a947916d 100644 --- a/lib/sanitizer_common/sanitizer_win_dll_thunk.cc +++ b/lib/sanitizer_common/sanitizer_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win_dll_thunk.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file defines a family of thunks that should be statically linked into diff --git a/lib/sanitizer_common/sanitizer_win_dll_thunk.h b/lib/sanitizer_common/sanitizer_win_dll_thunk.h index 2f9ebdaa6..48c73c4c9 100644 --- a/lib/sanitizer_common/sanitizer_win_dll_thunk.h +++ b/lib/sanitizer_common/sanitizer_win_dll_thunk.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_dll_thunk.h -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This header provide helper macros to delegate calls to the shared runtime diff --git a/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc b/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc index f8f916473..6bcde34e2 100644 --- a/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc +++ b/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- santizer_win_dynamic_runtime_thunk.cc -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win_weak_interception.cc b/lib/sanitizer_common/sanitizer_win_weak_interception.cc index 5711f5dc8..b1ac44d75 100644 --- a/lib/sanitizer_common/sanitizer_win_weak_interception.cc +++ b/lib/sanitizer_common/sanitizer_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win_weak_interception.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in the sanitizer when it is implemented as a diff --git a/lib/sanitizer_common/sanitizer_win_weak_interception.h b/lib/sanitizer_common/sanitizer_win_weak_interception.h index 5b122971d..5e4d8b8de 100644 --- a/lib/sanitizer_common/sanitizer_win_weak_interception.h +++ b/lib/sanitizer_common/sanitizer_win_weak_interception.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_weak_interception.h ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This header provide helper macros to delegate calls of weak functions to the diff --git a/lib/sanitizer_common/scripts/gen_dynamic_list.py b/lib/sanitizer_common/scripts/gen_dynamic_list.py index 4a9c7af95..095fe68c8 100755 --- a/lib/sanitizer_common/scripts/gen_dynamic_list.py +++ b/lib/sanitizer_common/scripts/gen_dynamic_list.py @@ -1,10 +1,9 @@ #!/usr/bin/env python #===- lib/sanitizer_common/scripts/gen_dynamic_list.py ---------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # diff --git a/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc b/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc index b25a53d73..8150b7a09 100644 --- a/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc +++ b/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolize.cc ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,8 +37,11 @@ bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset, { llvm::raw_string_ostream OS(Result); llvm::symbolize::DIPrinter Printer(OS); - auto ResOrErr = - getDefaultSymbolizer()->symbolizeInlinedCode(ModuleName, ModuleOffset); + // TODO: it is neccessary to set proper SectionIndex here. + // object::SectionedAddress::UndefSection works for only absolute addresses. + auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode( + ModuleName, + {ModuleOffset, llvm::object::SectionedAddress::UndefSection}); Printer << (ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo()); } return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s", @@ -52,8 +54,11 @@ bool __sanitizer_symbolize_data(const char *ModuleName, uint64_t ModuleOffset, { llvm::raw_string_ostream OS(Result); llvm::symbolize::DIPrinter Printer(OS); - auto ResOrErr = - getDefaultSymbolizer()->symbolizeData(ModuleName, ModuleOffset); + // TODO: it is neccessary to set proper SectionIndex here. + // object::SectionedAddress::UndefSection works for only absolute addresses. + auto ResOrErr = getDefaultSymbolizer()->symbolizeData( + ModuleName, + {ModuleOffset, llvm::object::SectionedAddress::UndefSection}); Printer << (ResOrErr ? ResOrErr.get() : llvm::DIGlobal()); } return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s", diff --git a/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc b/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc index 66d089a0e..c85ebe5e2 100644 --- a/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc +++ b/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc @@ -1,9 +1,8 @@ //===-- sanitizer_wrappers.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh b/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh index 788cef85a..5c77bea83 100755 --- a/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh +++ b/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh @@ -9,8 +9,8 @@ if [ "$#" -le 1 ]; then usage fi -AR=$(readlink -f $AR) -LINK=$(readlink -f $LINK) +[[ $AR == /* ]] || AR=$PWD/$AR +[[ $LINK == /* ]] || LINK=$PWD/$LINK INPUTS= OUTPUT= diff --git a/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt index beee0acf4..e20fdc7c3 100644 --- a/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt +++ b/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt @@ -36,6 +36,7 @@ __umoddi3 U _exit U abort U access U +bcmp U calloc U catclose U catgets U @@ -132,6 +133,7 @@ strtol U strtold_l U strtoll_l U strtoull_l U +syscall U tcgetattr U uname U ungetc U diff --git a/lib/sanitizer_common/tests/CMakeLists.txt b/lib/sanitizer_common/tests/CMakeLists.txt index 21ffe2528..fd29d178b 100644 --- a/lib/sanitizer_common/tests/CMakeLists.txt +++ b/lib/sanitizer_common/tests/CMakeLists.txt @@ -59,6 +59,8 @@ set(SANITIZER_TEST_CFLAGS_COMMON -Wno-non-virtual-dtor -Wno-gnu-zero-variadic-macro-arguments) +set(SANITIZER_TEST_LINK_FLAGS_COMMON ${COMPILER_RT_UNITTEST_LINK_FLAGS}) + # -gline-tables-only must be enough for these tests, so use it if possible. if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang") list(APPEND SANITIZER_TEST_CFLAGS_COMMON -gline-tables-only) diff --git a/lib/sanitizer_common/tests/sanitizer_allocator_test.cc b/lib/sanitizer_common/tests/sanitizer_allocator_test.cc index 3123a1d5a..5d99bf458 100644 --- a/lib/sanitizer_common/tests/sanitizer_allocator_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_allocator_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_test.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -144,7 +143,6 @@ static const u64 kAddressSpaceSize = 1ULL << 32; #endif static const uptr kRegionSizeLog = FIRST_32_SECOND_64(20, 24); -static const uptr kFlatByteMapSize = kAddressSpaceSize >> kRegionSizeLog; template <typename AddressSpaceViewTy> struct AP32Compact { @@ -154,7 +152,6 @@ struct AP32Compact { typedef CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = ::kRegionSizeLog; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = FlatByteMap<kFlatByteMapSize, AddressSpaceView>; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -194,7 +191,7 @@ template <class Allocator> void TestSizeClassAllocator() { Allocator *a = new Allocator; a->Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<Allocator> cache; + typename Allocator::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); @@ -300,7 +297,6 @@ struct AP32SeparateBatches { typedef DefaultSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = ::kRegionSizeLog; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = FlatByteMap<kFlatByteMapSize, AddressSpaceView>; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; @@ -319,7 +315,7 @@ template <class Allocator> void SizeClassAllocatorMetadataStress() { Allocator *a = new Allocator; a->Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<Allocator> cache; + typename Allocator::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); @@ -373,7 +369,7 @@ template <class Allocator> void SizeClassAllocatorGetBlockBeginStress(u64 TotalSize) { Allocator *a = new Allocator; a->Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<Allocator> cache; + typename Allocator::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); @@ -450,7 +446,7 @@ TEST(SanitizerCommon, SizeClassAllocator64MapUnmapCallback) { Allocator64WithCallBack *a = new Allocator64WithCallBack; a->Init(kReleaseToOSIntervalNever); EXPECT_EQ(TestMapUnmapCallback::map_count, 1); // Allocator state. - SizeClassAllocatorLocalCache<Allocator64WithCallBack> cache; + typename Allocator64WithCallBack::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); AllocatorStats stats; @@ -475,7 +471,6 @@ struct AP32WithCallback { typedef CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = ::kRegionSizeLog; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = FlatByteMap<kFlatByteMapSize, AddressSpaceView>; typedef TestMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -487,7 +482,7 @@ TEST(SanitizerCommon, SizeClassAllocator32MapUnmapCallback) { Allocator32WithCallBack *a = new Allocator32WithCallBack; a->Init(kReleaseToOSIntervalNever); EXPECT_EQ(TestMapUnmapCallback::map_count, 0); - SizeClassAllocatorLocalCache<Allocator32WithCallBack> cache; + Allocator32WithCallBack::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); AllocatorStats stats; @@ -521,7 +516,7 @@ TEST(SanitizerCommon, LargeMmapAllocatorMapUnmapCallback) { TEST(SanitizerCommon, SizeClassAllocator64Overflow) { Allocator64 a; a.Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<Allocator64> cache; + Allocator64::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); AllocatorStats stats; @@ -620,17 +615,14 @@ TEST(SanitizerCommon, LargeMmapAllocator) { a.Deallocate(&stats, p); } -template -<class PrimaryAllocator, class SecondaryAllocator, class AllocatorCache> +template <class PrimaryAllocator> void TestCombinedAllocator() { - typedef - CombinedAllocator<PrimaryAllocator, AllocatorCache, SecondaryAllocator> - Allocator; + typedef CombinedAllocator<PrimaryAllocator> Allocator; Allocator *a = new Allocator; a->Init(kReleaseToOSIntervalNever); std::mt19937 r; - AllocatorCache cache; + typename Allocator::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); a->InitCache(&cache); @@ -691,42 +683,32 @@ void TestCombinedAllocator() { #if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, CombinedAllocator64) { - TestCombinedAllocator<Allocator64, - LargeMmapAllocator<>, - SizeClassAllocatorLocalCache<Allocator64> > (); + TestCombinedAllocator<Allocator64>(); } TEST(SanitizerCommon, CombinedAllocator64Dynamic) { - TestCombinedAllocator<Allocator64Dynamic, - LargeMmapAllocator<>, - SizeClassAllocatorLocalCache<Allocator64Dynamic> > (); + TestCombinedAllocator<Allocator64Dynamic>(); } #if !SANITIZER_ANDROID TEST(SanitizerCommon, CombinedAllocator64Compact) { - TestCombinedAllocator<Allocator64Compact, - LargeMmapAllocator<>, - SizeClassAllocatorLocalCache<Allocator64Compact> > (); + TestCombinedAllocator<Allocator64Compact>(); } #endif TEST(SanitizerCommon, CombinedAllocator64VeryCompact) { - TestCombinedAllocator<Allocator64VeryCompact, - LargeMmapAllocator<>, - SizeClassAllocatorLocalCache<Allocator64VeryCompact> > (); + TestCombinedAllocator<Allocator64VeryCompact>(); } #endif TEST(SanitizerCommon, CombinedAllocator32Compact) { - TestCombinedAllocator<Allocator32Compact, - LargeMmapAllocator<>, - SizeClassAllocatorLocalCache<Allocator32Compact> > (); + TestCombinedAllocator<Allocator32Compact>(); } -template <class AllocatorCache> +template <class Allocator> void TestSizeClassAllocatorLocalCache() { + using AllocatorCache = typename Allocator::AllocatorCache; AllocatorCache cache; - typedef typename AllocatorCache::Allocator Allocator; Allocator *a = new Allocator(); a->Init(kReleaseToOSIntervalNever); @@ -762,35 +744,30 @@ void TestSizeClassAllocatorLocalCache() { // to run them all at the same time. FIXME: Make them not flaky and reenable. #if !SANITIZER_WINDOWS TEST(SanitizerCommon, SizeClassAllocator64LocalCache) { - TestSizeClassAllocatorLocalCache< - SizeClassAllocatorLocalCache<Allocator64> >(); + TestSizeClassAllocatorLocalCache<Allocator64>(); } TEST(SanitizerCommon, SizeClassAllocator64DynamicLocalCache) { - TestSizeClassAllocatorLocalCache< - SizeClassAllocatorLocalCache<Allocator64Dynamic> >(); + TestSizeClassAllocatorLocalCache<Allocator64Dynamic>(); } #if !SANITIZER_ANDROID TEST(SanitizerCommon, SizeClassAllocator64CompactLocalCache) { - TestSizeClassAllocatorLocalCache< - SizeClassAllocatorLocalCache<Allocator64Compact> >(); + TestSizeClassAllocatorLocalCache<Allocator64Compact>(); } #endif TEST(SanitizerCommon, SizeClassAllocator64VeryCompactLocalCache) { - TestSizeClassAllocatorLocalCache< - SizeClassAllocatorLocalCache<Allocator64VeryCompact> >(); + TestSizeClassAllocatorLocalCache<Allocator64VeryCompact>(); } #endif #endif TEST(SanitizerCommon, SizeClassAllocator32CompactLocalCache) { - TestSizeClassAllocatorLocalCache< - SizeClassAllocatorLocalCache<Allocator32Compact> >(); + TestSizeClassAllocatorLocalCache<Allocator32Compact>(); } #if SANITIZER_CAN_USE_ALLOCATOR64 -typedef SizeClassAllocatorLocalCache<Allocator64> AllocatorCache; +typedef Allocator64::AllocatorCache AllocatorCache; static AllocatorCache static_allocator_cache; void *AllocatorLeakTestWorker(void *arg) { @@ -909,7 +886,7 @@ template <class Allocator> void TestSizeClassAllocatorIteration() { Allocator *a = new Allocator; a->Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<Allocator> cache; + typename Allocator::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); @@ -1032,7 +1009,7 @@ TEST(SanitizerCommon, LargeMmapAllocatorBlockBegin) { // Don't test OOM conditions on Win64 because it causes other tests on the same // machine to OOM. #if SANITIZER_CAN_USE_ALLOCATOR64 && !SANITIZER_WINDOWS64 && !SANITIZER_ANDROID -typedef SizeClassMap<3, 4, 8, 63, 128, 16> SpecialSizeClassMap; +typedef __sanitizer::SizeClassMap<3, 4, 8, 63, 128, 16> SpecialSizeClassMap; template <typename AddressSpaceViewTy = LocalAddressSpaceView> struct AP64_SpecialSizeClassMap { static const uptr kSpaceBeg = kAllocatorSpace; @@ -1052,7 +1029,7 @@ TEST(SanitizerCommon, SizeClassAllocator64PopulateFreeListOOM) { kAllocatorSize / SpecialSizeClassMap::kNumClassesRounded; SpecialAllocator64 *a = new SpecialAllocator64; a->Init(kReleaseToOSIntervalNever); - SizeClassAllocatorLocalCache<SpecialAllocator64> cache; + SpecialAllocator64::AllocatorCache cache; memset(&cache, 0, sizeof(cache)); cache.Init(0); diff --git a/lib/sanitizer_common/tests/sanitizer_allocator_testlib.cc b/lib/sanitizer_common/tests/sanitizer_allocator_testlib.cc index d2920d8f7..e7d482fec 100644 --- a/lib/sanitizer_common/tests/sanitizer_allocator_testlib.cc +++ b/lib/sanitizer_common/tests/sanitizer_allocator_testlib.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_testlib.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Malloc replacement library based on CombinedAllocator. @@ -50,10 +49,8 @@ struct __AP64 { namespace { typedef SizeClassAllocator64<__AP64> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; static Allocator allocator; static bool global_inited; diff --git a/lib/sanitizer_common/tests/sanitizer_atomic_test.cc b/lib/sanitizer_common/tests/sanitizer_atomic_test.cc index 56bcd35c8..37ba0fa3f 100644 --- a/lib/sanitizer_common/tests/sanitizer_atomic_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_atomic_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_bitvector_test.cc b/lib/sanitizer_common/tests/sanitizer_bitvector_test.cc index 669365b80..9f605037a 100644 --- a/lib/sanitizer_common/tests/sanitizer_bitvector_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_bitvector_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_bitvector_test.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_bvgraph_test.cc b/lib/sanitizer_common/tests/sanitizer_bvgraph_test.cc index 3b39f8dd7..955b723c6 100644 --- a/lib/sanitizer_common/tests/sanitizer_bvgraph_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_bvgraph_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_bvgraph_test.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_common_test.cc b/lib/sanitizer_common/tests/sanitizer_common_test.cc index 6b091de60..2350de943 100644 --- a/lib/sanitizer_common/tests/sanitizer_common_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_common_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -439,4 +438,12 @@ TEST(SanitizerCommon, ReservedAddressRangeUnmap) { EXPECT_DEATH(address_range.Unmap(base_addr + (PageSize * 2), PageSize), ".*"); } +// Windows has no working ReadBinaryName. +#if !SANITIZER_WINDOWS +TEST(SanitizerCommon, ReadBinaryNameCached) { + char buf[256]; + EXPECT_NE((uptr)0, ReadBinaryNameCached(buf, sizeof(buf))); +} +#endif + } // namespace __sanitizer diff --git a/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc b/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc index 7835eef76..f68bb70df 100644 --- a/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector_test.cc -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_flags_test.cc b/lib/sanitizer_common/tests/sanitizer_flags_test.cc index f3fe139e6..cfe90ef45 100644 --- a/lib/sanitizer_common/tests/sanitizer_flags_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_flags_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_flags_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,6 +16,7 @@ #include "sanitizer_common/sanitizer_allocator_internal.h" #include "gtest/gtest.h" +#include <stdint.h> #include <string.h> namespace __sanitizer { @@ -34,6 +34,9 @@ static void TestFlag(T start_value, const char *env, T final_value) { parser.ParseString(env); EXPECT_EQ(final_value, flag); + + // Reporting unrecognized flags is needed to reset them. + ReportUnrecognizedFlags(); } template <> @@ -108,6 +111,21 @@ TEST(SanitizerCommon, IntFlags) { "Invalid value for int option"); } +TEST(SanitizerCommon, LongLongIntFlags) { + s64 InitValue = -5; + s64 IntMin = INT64_MIN; + s64 IntMax = INT64_MAX; + TestFlag(InitValue, "flag_name=0", 0ll); + TestFlag(InitValue, "flag_name=42", 42ll); + TestFlag(InitValue, "flag_name=-42", -42ll); + + TestFlag(InitValue, "flag_name=-9223372036854775808", IntMin); + TestFlag(InitValue, "flag_name=9223372036854775807", IntMax); + + TestFlag(InitValue, "flag_name=-92233720368547758080000", IntMin); + TestFlag(InitValue, "flag_name=92233720368547758070000", IntMax); +} + TEST(SanitizerCommon, StrFlags) { TestFlag("zzz", 0, "zzz"); TestFlag("zzz", "flag_name=", ""); diff --git a/lib/sanitizer_common/tests/sanitizer_format_interceptor_test.cc b/lib/sanitizer_common/tests/sanitizer_format_interceptor_test.cc index 2f0494f82..9f70fbc52 100644 --- a/lib/sanitizer_common/tests/sanitizer_format_interceptor_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_format_interceptor_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_format_interceptor_test.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_ioctl_test.cc b/lib/sanitizer_common/tests/sanitizer_ioctl_test.cc index 6e2a20b85..738046aba 100644 --- a/lib/sanitizer_common/tests/sanitizer_ioctl_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_ioctl_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_ioctl_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_libc_test.cc b/lib/sanitizer_common/tests/sanitizer_libc_test.cc index 2f61601cd..d8f475991 100644 --- a/lib/sanitizer_common/tests/sanitizer_libc_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_libc_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_libc_test.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Tests for sanitizer_libc.h. diff --git a/lib/sanitizer_common/tests/sanitizer_linux_test.cc b/lib/sanitizer_common/tests/sanitizer_linux_test.cc index fbac9cc14..a5ce5a2c3 100644 --- a/lib/sanitizer_common/tests/sanitizer_linux_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_linux_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_list_test.cc b/lib/sanitizer_common/tests/sanitizer_list_test.cc index ede9771cb..7dd28ee05 100644 --- a/lib/sanitizer_common/tests/sanitizer_list_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_list_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_list_test.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_mutex_test.cc b/lib/sanitizer_common/tests/sanitizer_mutex_test.cc index d14e7c2fb..ef1c5fa46 100644 --- a/lib/sanitizer_common/tests/sanitizer_mutex_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_mutex_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_mutex_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_nolibc_test.cc b/lib/sanitizer_common/tests/sanitizer_nolibc_test.cc index d0d5a5e13..fdab29692 100644 --- a/lib/sanitizer_common/tests/sanitizer_nolibc_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_nolibc_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_nolibc_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_nolibc_test_main.cc b/lib/sanitizer_common/tests/sanitizer_nolibc_test_main.cc index e761f00c5..70028506a 100644 --- a/lib/sanitizer_common/tests/sanitizer_nolibc_test_main.cc +++ b/lib/sanitizer_common/tests/sanitizer_nolibc_test_main.cc @@ -1,9 +1,8 @@ //===-- sanitizer_nolibc_test_main.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_posix_test.cc b/lib/sanitizer_common/tests/sanitizer_posix_test.cc index b7cca8362..6ceae7d77 100644 --- a/lib/sanitizer_common/tests/sanitizer_posix_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_posix_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_posix_test.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_printf_test.cc b/lib/sanitizer_common/tests/sanitizer_printf_test.cc index 75fe66678..4f86976c5 100644 --- a/lib/sanitizer_common/tests/sanitizer_printf_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_printf_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_printf_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc b/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc index 22052d9a4..37ab3d96d 100644 --- a/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_test.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_pthread_wrappers.h b/lib/sanitizer_common/tests/sanitizer_pthread_wrappers.h index b7d784c25..f806ee1ea 100644 --- a/lib/sanitizer_common/tests/sanitizer_pthread_wrappers.h +++ b/lib/sanitizer_common/tests/sanitizer_pthread_wrappers.h @@ -1,9 +1,8 @@ //===-- sanitizer_pthread_wrappers.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_quarantine_test.cc b/lib/sanitizer_common/tests/sanitizer_quarantine_test.cc index 23ed5f97a..4088119a9 100644 --- a/lib/sanitizer_common/tests/sanitizer_quarantine_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_quarantine_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_quarantine_test.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_ring_buffer_test.cc b/lib/sanitizer_common/tests/sanitizer_ring_buffer_test.cc index 80aa57c52..e10cd3604 100644 --- a/lib/sanitizer_common/tests/sanitizer_ring_buffer_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_ring_buffer_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_vector_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cc b/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cc index 513432fac..24f6fcf77 100644 --- a/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepot_test.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cc b/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cc index 405f8d86e..b6d1bd16b 100644 --- a/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_printer_test.cc ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc b/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc index ba9f4fd69..771a3e4d9 100644 --- a/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_test.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,18 +20,15 @@ class FastUnwindTest : public ::testing::Test { protected: virtual void SetUp(); virtual void TearDown(); - bool TryFastUnwind(uptr max_depth) { - if (!StackTrace::WillUseFastUnwind(true)) - return false; - trace.Unwind(max_depth, start_pc, (uptr)&fake_stack[0], 0, fake_top, - fake_bottom, true); - return true; - } + + void UnwindFast(); void *mapping; uhwptr *fake_stack; const uptr fake_stack_size = 10; uhwptr start_pc; + + uhwptr fake_bp; uhwptr fake_top; uhwptr fake_bottom; BufferedStackTrace trace; @@ -59,10 +55,11 @@ void FastUnwindTest::SetUp() { // Mark the last fp point back up to terminate the stack trace. fake_stack[RoundDownTo(fake_stack_size - 1, 2)] = (uhwptr)&fake_stack[0]; - // Top is two slots past the end because FastUnwindStack subtracts two. + // Top is two slots past the end because UnwindFast subtracts two. fake_top = (uhwptr)&fake_stack[fake_stack_size + 2]; - // Bottom is one slot before the start because FastUnwindStack uses >. + // Bottom is one slot before the start because UnwindFast uses >. fake_bottom = (uhwptr)mapping; + fake_bp = (uptr)&fake_stack[0]; start_pc = PC(0); } @@ -71,9 +68,14 @@ void FastUnwindTest::TearDown() { UnmapOrDie(mapping, 2 * ps); } +#if SANITIZER_CAN_FAST_UNWIND + +void FastUnwindTest::UnwindFast() { + trace.UnwindFast(start_pc, fake_bp, fake_top, fake_bottom, kStackTraceMax); +} + TEST_F(FastUnwindTest, Basic) { - if (!TryFastUnwind(kStackTraceMax)) - return; + UnwindFast(); // Should get all on-stack retaddrs and start_pc. EXPECT_EQ(6U, trace.size); EXPECT_EQ(start_pc, trace.trace[0]); @@ -86,8 +88,7 @@ TEST_F(FastUnwindTest, Basic) { TEST_F(FastUnwindTest, FramePointerLoop) { // Make one fp point to itself. fake_stack[4] = (uhwptr)&fake_stack[4]; - if (!TryFastUnwind(kStackTraceMax)) - return; + UnwindFast(); // Should get all on-stack retaddrs up to the 4th slot and start_pc. EXPECT_EQ(4U, trace.size); EXPECT_EQ(start_pc, trace.trace[0]); @@ -99,8 +100,7 @@ TEST_F(FastUnwindTest, FramePointerLoop) { TEST_F(FastUnwindTest, MisalignedFramePointer) { // Make one fp misaligned. fake_stack[4] += 3; - if (!TryFastUnwind(kStackTraceMax)) - return; + UnwindFast(); // Should get all on-stack retaddrs up to the 4th slot and start_pc. EXPECT_EQ(4U, trace.size); EXPECT_EQ(start_pc, trace.trace[0]); @@ -110,16 +110,14 @@ TEST_F(FastUnwindTest, MisalignedFramePointer) { } TEST_F(FastUnwindTest, OneFrameStackTrace) { - if (!TryFastUnwind(1)) - return; + trace.Unwind(start_pc, fake_bp, nullptr, true, 1); EXPECT_EQ(1U, trace.size); EXPECT_EQ(start_pc, trace.trace[0]); EXPECT_EQ((uhwptr)&fake_stack[0], trace.top_frame_bp); } TEST_F(FastUnwindTest, ZeroFramesStackTrace) { - if (!TryFastUnwind(0)) - return; + trace.Unwind(start_pc, fake_bp, nullptr, true, 0); EXPECT_EQ(0U, trace.size); EXPECT_EQ(0U, trace.top_frame_bp); } @@ -129,8 +127,7 @@ TEST_F(FastUnwindTest, FPBelowPrevFP) { // current FP. fake_stack[0] = (uhwptr)&fake_stack[-50]; fake_stack[1] = PC(1); - if (!TryFastUnwind(3)) - return; + UnwindFast(); EXPECT_EQ(2U, trace.size); EXPECT_EQ(PC(0), trace.trace[0]); EXPECT_EQ(PC(1), trace.trace[1]); @@ -139,8 +136,7 @@ TEST_F(FastUnwindTest, FPBelowPrevFP) { TEST_F(FastUnwindTest, CloseToZeroFrame) { // Make one pc a NULL pointer. fake_stack[5] = 0x0; - if (!TryFastUnwind(kStackTraceMax)) - return; + UnwindFast(); // The stack should be truncated at the NULL pointer (and not include it). EXPECT_EQ(3U, trace.size); EXPECT_EQ(start_pc, trace.trace[0]); @@ -149,16 +145,16 @@ TEST_F(FastUnwindTest, CloseToZeroFrame) { } } +#endif // SANITIZER_CAN_FAST_UNWIND + TEST(SlowUnwindTest, ShortStackTrace) { - if (StackTrace::WillUseFastUnwind(false)) - return; BufferedStackTrace stack; uptr pc = StackTrace::GetCurrentPc(); uptr bp = GET_CURRENT_FRAME(); - stack.Unwind(0, pc, bp, 0, 0, 0, false); + stack.Unwind(pc, bp, nullptr, false, /*max_depth=*/0); EXPECT_EQ(0U, stack.size); EXPECT_EQ(0U, stack.top_frame_bp); - stack.Unwind(1, pc, bp, 0, 0, 0, false); + stack.Unwind(pc, bp, nullptr, false, /*max_depth=*/1); EXPECT_EQ(1U, stack.size); EXPECT_EQ(pc, stack.trace[0]); EXPECT_EQ(bp, stack.top_frame_bp); diff --git a/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cc b/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cc index 802af392c..98e64d870 100644 --- a/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_test.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc b/lib/sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc index d8be2afb1..033170e7a 100644 --- a/lib/sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc +++ b/lib/sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_testlib.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Dynamic library to test StopTheWorld functionality. diff --git a/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc b/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc index 224ab0538..d64379a44 100644 --- a/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_suppressions_test.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -132,4 +131,10 @@ TEST_F(SuppressionContextTest, HasSuppressionType) { EXPECT_FALSE(ctx_.HasSuppressionType("signal")); } +TEST_F(SuppressionContextTest, RegressionTestForBufferOverflowInSuppressions) { + EXPECT_DEATH(ctx_.Parse("race"), "failed to parse suppressions"); + EXPECT_DEATH(ctx_.Parse("foo"), "failed to parse suppressions"); +} + + } // namespace __sanitizer diff --git a/lib/sanitizer_common/tests/sanitizer_symbolizer_test.cc b/lib/sanitizer_common/tests/sanitizer_symbolizer_test.cc index 4c4d2a8c3..e6bdeaa56 100644 --- a/lib/sanitizer_common/tests/sanitizer_symbolizer_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_symbolizer_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_test.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_test_config.h b/lib/sanitizer_common/tests/sanitizer_test_config.h index bdf614606..ed35028a3 100644 --- a/lib/sanitizer_common/tests/sanitizer_test_config.h +++ b/lib/sanitizer_common/tests/sanitizer_test_config.h @@ -1,9 +1,8 @@ //===-- sanitizer_test_config.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_test_main.cc b/lib/sanitizer_common/tests/sanitizer_test_main.cc index 0da886120..8edee7458 100644 --- a/lib/sanitizer_common/tests/sanitizer_test_main.cc +++ b/lib/sanitizer_common/tests/sanitizer_test_main.cc @@ -1,9 +1,8 @@ //===-- sanitizer_test_main.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_test_utils.h b/lib/sanitizer_common/tests/sanitizer_test_utils.h index 5c1f8ad48..525b1e485 100644 --- a/lib/sanitizer_common/tests/sanitizer_test_utils.h +++ b/lib/sanitizer_common/tests/sanitizer_test_utils.h @@ -1,9 +1,8 @@ //===-- sanitizer_test_utils.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cc b/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cc index f8b8c12d4..09c01d6c8 100644 --- a/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_thread_registry_test.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -67,7 +66,7 @@ static void MarkUidAsPresent(ThreadContextBase *tctx, void *arg) { static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) { // Create and start a main thread. EXPECT_EQ(0U, registry->CreateThread(get_uid(0), true, -1, 0)); - registry->StartThread(0, 0, false, 0); + registry->StartThread(0, 0, ThreadType::Regular, 0); // Create a bunch of threads. for (u32 i = 1; i <= 10; i++) { EXPECT_EQ(i, registry->CreateThread(get_uid(i), is_detached(i), 0, 0)); @@ -75,7 +74,7 @@ static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) { CheckThreadQuantity(registry, 11, 1, 11); // Start some of them. for (u32 i = 1; i <= 5; i++) { - registry->StartThread(i, 0, false, 0); + registry->StartThread(i, 0, ThreadType::Regular, 0); } CheckThreadQuantity(registry, 11, 6, 11); // Finish, create and start more threads. @@ -85,7 +84,7 @@ static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) { registry->JoinThread(i, 0); } for (u32 i = 6; i <= 10; i++) { - registry->StartThread(i, 0, false, 0); + registry->StartThread(i, 0, ThreadType::Regular, 0); } std::vector<u32> new_tids; for (u32 i = 11; i <= 15; i++) { @@ -112,7 +111,7 @@ static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) { } for (u32 i = 0; i < new_tids.size(); i++) { u32 tid = new_tids[i]; - registry->StartThread(tid, 0, false, 0); + registry->StartThread(tid, 0, ThreadType::Regular, 0); registry->DetachThread(tid, 0); registry->FinishThread(tid); } @@ -189,7 +188,8 @@ void *RunThread(void *arg) { tids.push_back( args->registry->CreateThread(0, false, 0, (void*)args->shard)); for (int i = 0; i < kThreadsPerShard; i++) - args->registry->StartThread(tids[i], 0, false, (void*)args->shard); + args->registry->StartThread(tids[i], 0, ThreadType::Regular, + (void*)args->shard); for (int i = 0; i < kThreadsPerShard; i++) args->registry->FinishThread(tids[i]); for (int i = 0; i < kThreadsPerShard; i++) @@ -200,7 +200,7 @@ void *RunThread(void *arg) { static void ThreadedTestRegistry(ThreadRegistry *registry) { // Create and start a main thread. EXPECT_EQ(0U, registry->CreateThread(0, true, -1, 0)); - registry->StartThread(0, 0, false, 0); + registry->StartThread(0, 0, ThreadType::Regular, 0); pthread_t threads[kNumShards]; RunThreadArgs args[kNumShards]; for (int i = 0; i < kNumShards; i++) { diff --git a/lib/sanitizer_common/tests/sanitizer_type_traits_test.cc b/lib/sanitizer_common/tests/sanitizer_type_traits_test.cc index 0dce02fac..ccefeb686 100644 --- a/lib/sanitizer_common/tests/sanitizer_type_traits_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_type_traits_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_type_traits_test.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,3 +25,8 @@ TEST(SanitizerCommon, IsSame) { ASSERT_FALSE((is_same<uptr, sptr>::value)); ASSERT_FALSE((is_same<uptr, const uptr>::value)); } + +TEST(SanitizerCommon, Conditional) { + ASSERT_TRUE((is_same<int, conditional<true, int, double>::type>::value)); + ASSERT_TRUE((is_same<double, conditional<false, int, double>::type>::value)); +} diff --git a/lib/sanitizer_common/tests/sanitizer_vector_test.cc b/lib/sanitizer_common/tests/sanitizer_vector_test.cc index 59fbf3968..5d96e9b94 100644 --- a/lib/sanitizer_common/tests/sanitizer_vector_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_vector_test.cc @@ -1,9 +1,8 @@ //===-- sanitizer_vector_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/scudo/CMakeLists.txt b/lib/scudo/CMakeLists.txt index 79f69e934..bbb8a1a9c 100644 --- a/lib/scudo/CMakeLists.txt +++ b/lib/scudo/CMakeLists.txt @@ -37,7 +37,7 @@ if (FUCHSIA) list(APPEND SCUDO_CFLAGS -nostdinc++) list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -nostdlib++) else() - list(APPEND SCUDO_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY}) + list(APPEND SCUDO_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES}) list(APPEND SCUDO_OBJECT_LIBS RTSanitizerCommonCoverage RTSanitizerCommonSymbolizer diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp index fb04fb281..172975eb9 100644 --- a/lib/scudo/scudo_allocator.cpp +++ b/lib/scudo/scudo_allocator.cpp @@ -1,9 +1,8 @@ //===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -588,11 +587,11 @@ NOINLINE void Allocator::performSanityChecks() { } // Opportunistic RSS limit check. This will update the RSS limit status, if -// it can, every 100ms, otherwise it will just return the current one. +// it can, every 250ms, otherwise it will just return the current one. NOINLINE bool Allocator::isRssLimitExceeded() { u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); const u64 CurrentCheck = MonotonicNanoTime(); - if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL))) + if (LIKELY(CurrentCheck < LastCheck + (250ULL * 1000000ULL))) return atomic_load_relaxed(&RssLimitExceeded); if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, CurrentCheck, memory_order_relaxed)) diff --git a/lib/scudo/scudo_allocator.h b/lib/scudo/scudo_allocator.h index 814bb08ab..0efa5c520 100644 --- a/lib/scudo/scudo_allocator.h +++ b/lib/scudo/scudo_allocator.h @@ -1,9 +1,8 @@ //===-- scudo_allocator.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -85,12 +84,6 @@ struct AP64 { }; typedef SizeClassAllocator64<AP64> PrimaryT; #else -static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; -# if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap<NumRegions> ByteMap; -# elif SANITIZER_WORDSIZE == 64 -typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; -# endif // SANITIZER_WORDSIZE struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; @@ -98,7 +91,6 @@ struct AP32 { typedef __scudo::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = RegionSizeLog; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __scudo::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator32FlagMasks::kRandomShuffleChunks | @@ -108,11 +100,13 @@ typedef SizeClassAllocator32<AP32> PrimaryT; #endif // SANITIZER_CAN_USE_ALLOCATOR64 #include "scudo_allocator_secondary.h" -#include "scudo_allocator_combined.h" -typedef SizeClassAllocatorLocalCache<PrimaryT> AllocatorCacheT; typedef LargeMmapAllocator SecondaryT; -typedef CombinedAllocator<PrimaryT, AllocatorCacheT, SecondaryT> BackendT; + +#include "scudo_allocator_combined.h" + +typedef CombinedAllocator BackendT; +typedef CombinedAllocator::AllocatorCache AllocatorCacheT; void initScudo(); diff --git a/lib/scudo/scudo_allocator_combined.h b/lib/scudo/scudo_allocator_combined.h index 6e40660ba..d61cc9ec1 100644 --- a/lib/scudo/scudo_allocator_combined.h +++ b/lib/scudo/scudo_allocator_combined.h @@ -1,9 +1,8 @@ //===-- scudo_allocator_combined.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,10 +18,11 @@ # error "This file must be included inside scudo_allocator.h." #endif -template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator> class CombinedAllocator { public: + using PrimaryAllocator = PrimaryT; + using SecondaryAllocator = SecondaryT; + using AllocatorCache = typename PrimaryAllocator::AllocatorCache; void init(s32 ReleaseToOSIntervalMs) { Primary.Init(ReleaseToOSIntervalMs); Secondary.Init(); diff --git a/lib/scudo/scudo_allocator_secondary.h b/lib/scudo/scudo_allocator_secondary.h index ff6246e25..151ff9931 100644 --- a/lib/scudo/scudo_allocator_secondary.h +++ b/lib/scudo/scudo_allocator_secondary.h @@ -1,9 +1,8 @@ //===-- scudo_allocator_secondary.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_crc32.cpp b/lib/scudo/scudo_crc32.cpp index a267dc4e3..87473505f 100644 --- a/lib/scudo/scudo_crc32.cpp +++ b/lib/scudo/scudo_crc32.cpp @@ -1,9 +1,8 @@ //===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_crc32.h b/lib/scudo/scudo_crc32.h index e89e430f4..bad15a929 100644 --- a/lib/scudo/scudo_crc32.h +++ b/lib/scudo/scudo_crc32.h @@ -1,9 +1,8 @@ //===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_errors.cpp b/lib/scudo/scudo_errors.cpp index d11e03cf9..34e57bf71 100644 --- a/lib/scudo/scudo_errors.cpp +++ b/lib/scudo/scudo_errors.cpp @@ -1,9 +1,8 @@ //===-- scudo_errors.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_errors.h b/lib/scudo/scudo_errors.h index 8b1af996b..258695c2c 100644 --- a/lib/scudo/scudo_errors.h +++ b/lib/scudo/scudo_errors.h @@ -1,9 +1,8 @@ //===-- scudo_errors.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_flags.cpp b/lib/scudo/scudo_flags.cpp index c012471a8..af8ae5b4c 100644 --- a/lib/scudo/scudo_flags.cpp +++ b/lib/scudo/scudo_flags.cpp @@ -1,9 +1,8 @@ //===-- scudo_flags.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_flags.h b/lib/scudo/scudo_flags.h index d4ae31031..483c79621 100644 --- a/lib/scudo/scudo_flags.h +++ b/lib/scudo/scudo_flags.h @@ -1,9 +1,8 @@ //===-- scudo_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_flags.inc b/lib/scudo/scudo_flags.inc index f180478fd..c124738c1 100644 --- a/lib/scudo/scudo_flags.inc +++ b/lib/scudo/scudo_flags.inc @@ -1,9 +1,8 @@ //===-- scudo_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -37,7 +36,9 @@ SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, "Size in bytes up to which chunks will be quarantined (if lower than" "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") -SCUDO_FLAG(bool, DeallocationTypeMismatch, true, +// Disable the deallocation type check by default on Android, it causes too many +// issues with third party libraries. +SCUDO_FLAG(bool, DeallocationTypeMismatch, !SANITIZER_ANDROID, "Report errors on malloc/delete, new/free, new/delete[], etc.") SCUDO_FLAG(bool, DeleteSizeMismatch, true, diff --git a/lib/scudo/scudo_interface_internal.h b/lib/scudo/scudo_interface_internal.h index 3e520a50c..75c63aa6d 100644 --- a/lib/scudo/scudo_interface_internal.h +++ b/lib/scudo/scudo_interface_internal.h @@ -1,9 +1,8 @@ //===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_malloc.cpp b/lib/scudo/scudo_malloc.cpp index eef776809..a72b861e2 100644 --- a/lib/scudo/scudo_malloc.cpp +++ b/lib/scudo/scudo_malloc.cpp @@ -1,9 +1,8 @@ //===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_new_delete.cpp b/lib/scudo/scudo_new_delete.cpp index daa3b47dc..03eef7f28 100644 --- a/lib/scudo/scudo_new_delete.cpp +++ b/lib/scudo/scudo_new_delete.cpp @@ -1,9 +1,8 @@ //===-- scudo_new_delete.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_platform.h b/lib/scudo/scudo_platform.h index 3a6f4be69..07d4b70fc 100644 --- a/lib/scudo/scudo_platform.h +++ b/lib/scudo/scudo_platform.h @@ -1,9 +1,8 @@ //===-- scudo_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -80,7 +79,7 @@ const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; #endif // SANITIZER_CAN_USE_ALLOCATOR64 #if !defined(SCUDO_SIZE_CLASS_MAP) -# define SCUDO_SIZE_CLASS_MAP Default +# define SCUDO_SIZE_CLASS_MAP Dense #endif #define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) diff --git a/lib/scudo/scudo_termination.cpp b/lib/scudo/scudo_termination.cpp index 4237d3bc1..6c7c0abc6 100644 --- a/lib/scudo/scudo_termination.cpp +++ b/lib/scudo/scudo_termination.cpp @@ -1,9 +1,8 @@ //===-- scudo_termination.cpp -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd.h b/lib/scudo/scudo_tsd.h index 2bd78716a..1d4e4e6f1 100644 --- a/lib/scudo/scudo_tsd.h +++ b/lib/scudo/scudo_tsd.h @@ -1,9 +1,8 @@ //===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_exclusive.cpp b/lib/scudo/scudo_tsd_exclusive.cpp index 74e797580..a203a74bb 100644 --- a/lib/scudo/scudo_tsd_exclusive.cpp +++ b/lib/scudo/scudo_tsd_exclusive.cpp @@ -1,9 +1,8 @@ //===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_exclusive.inc b/lib/scudo/scudo_tsd_exclusive.inc index 1fa9dcdfd..08e4d3af7 100644 --- a/lib/scudo/scudo_tsd_exclusive.inc +++ b/lib/scudo/scudo_tsd_exclusive.inc @@ -1,9 +1,8 @@ //===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_shared.cpp b/lib/scudo/scudo_tsd_shared.cpp index 8853894c0..9918a08be 100644 --- a/lib/scudo/scudo_tsd_shared.cpp +++ b/lib/scudo/scudo_tsd_shared.cpp @@ -1,9 +1,8 @@ //===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_shared.inc b/lib/scudo/scudo_tsd_shared.inc index 9dad756b5..8f3362dd3 100644 --- a/lib/scudo/scudo_tsd_shared.inc +++ b/lib/scudo/scudo_tsd_shared.inc @@ -1,9 +1,8 @@ //===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp index d5788d20c..5e76a4a30 100644 --- a/lib/scudo/scudo_utils.cpp +++ b/lib/scudo/scudo_utils.cpp @@ -1,9 +1,8 @@ //===-- scudo_utils.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h index 43448e083..a8dfbdeb3 100644 --- a/lib/scudo/scudo_utils.h +++ b/lib/scudo/scudo_utils.h @@ -1,9 +1,8 @@ //===-- scudo_utils.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/standalone/CMakeLists.txt b/lib/scudo/standalone/CMakeLists.txt new file mode 100644 index 000000000..922f98692 --- /dev/null +++ b/lib/scudo/standalone/CMakeLists.txt @@ -0,0 +1,98 @@ +add_compiler_rt_component(scudo_standalone) + +include_directories(../..) + +set(SCUDO_CFLAGS) + +list(APPEND SCUDO_CFLAGS + -Wall + -nostdinc++) + +# Remove -stdlib= which is unused when passing -nostdinc++. +string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + +append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding SCUDO_CFLAGS) + +append_list_if(COMPILER_RT_HAS_FVISIBILITY_HIDDEN_FLAG -fvisibility=hidden SCUDO_CFLAGS) + +if(COMPILER_RT_DEBUG) + list(APPEND SCUDO_CFLAGS -O0) +else() + list(APPEND SCUDO_CFLAGS -O3) +endif() + +set(SCUDO_LINK_FLAGS) + +list(APPEND SCUDO_LINK_FLAGS -Wl,-z,defs,-z,now,-z,relro) + +append_list_if(COMPILER_RT_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs SCUDO_LINK_FLAGS) + +if(ANDROID) +# Put the shared library in the global group. For more details, see +# android-changes-for-ndk-developers.md#changes-to-library-search-order + append_list_if(COMPILER_RT_HAS_Z_GLOBAL -Wl,-z,global SCUDO_LINK_FLAGS) +endif() + +set(SCUDO_SOURCES + checksum.cc + crc32_hw.cc + common.cc + flags.cc + flags_parser.cc + fuchsia.cc + linux.cc + report.cc + secondary.cc + string_utils.cc) + +# Enable the SSE 4.2 instruction set for crc32_hw.cc, if available. +if (COMPILER_RT_HAS_MSSE4_2_FLAG) + set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -msse4.2) +endif() + +# Enable the AArch64 CRC32 feature for crc32_hw.cc, if available. +# Note that it is enabled by default starting with armv8.1-a. +if (COMPILER_RT_HAS_MCRC_FLAG) + set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -mcrc) +endif() + +set(SCUDO_HEADERS + atomic_helpers.h + bytemap.h + checksum.h + flags.h + flags_parser.h + fuchsia.h + interface.h + internal_defs.h + linux.h + list.h + mutex.h + platform.h + release.h + report.h + secondary.h + size_class_map.h + stats.h + string_utils.h + vector.h) + +if(COMPILER_RT_HAS_SCUDO_STANDALONE) + add_compiler_rt_object_libraries(RTScudoStandalone + ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + CFLAGS ${SCUDO_CFLAGS}) + + add_compiler_rt_runtime(clang_rt.scudo_standalone + STATIC + ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + CFLAGS ${SCUDO_CFLAGS} + PARENT_TARGET scudo_standalone) + + if(COMPILER_RT_INCLUDE_TESTS) + add_subdirectory(tests) + endif() +endif() diff --git a/lib/scudo/standalone/atomic_helpers.h b/lib/scudo/standalone/atomic_helpers.h new file mode 100644 index 000000000..35d7369c1 --- /dev/null +++ b/lib/scudo/standalone/atomic_helpers.h @@ -0,0 +1,131 @@ +//===-- atomic_helpers.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ATOMIC_H_ +#define SCUDO_ATOMIC_H_ + +#include "internal_defs.h" + +namespace scudo { + +enum memory_order { + memory_order_relaxed = 0, + memory_order_consume = 1, + memory_order_acquire = 2, + memory_order_release = 3, + memory_order_acq_rel = 4, + memory_order_seq_cst = 5 +}; +COMPILER_CHECK(memory_order_relaxed == __ATOMIC_RELAXED); +COMPILER_CHECK(memory_order_consume == __ATOMIC_CONSUME); +COMPILER_CHECK(memory_order_acquire == __ATOMIC_ACQUIRE); +COMPILER_CHECK(memory_order_release == __ATOMIC_RELEASE); +COMPILER_CHECK(memory_order_acq_rel == __ATOMIC_ACQ_REL); +COMPILER_CHECK(memory_order_seq_cst == __ATOMIC_SEQ_CST); + +struct atomic_u8 { + typedef u8 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u16 { + typedef u16 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_s32 { + typedef s32 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u32 { + typedef u32 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u64 { + typedef u64 Type; + // On 32-bit platforms u64 is not necessarily aligned on 8 bytes. + ALIGNED(8) volatile Type ValDoNotUse; +}; + +struct atomic_uptr { + typedef uptr Type; + volatile Type ValDoNotUse; +}; + +template <typename T> +INLINE typename T::Type atomic_load(const volatile T *A, memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + typename T::Type V; + __atomic_load(&A->ValDoNotUse, &V, MO); + return V; +} + +template <typename T> +INLINE void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + __atomic_store(&A->ValDoNotUse, &V, MO); +} + +INLINE void atomic_thread_fence(memory_order) { __sync_synchronize(); } + +template <typename T> +INLINE typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + return __atomic_fetch_add(&A->ValDoNotUse, V, MO); +} + +template <typename T> +INLINE typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + return __atomic_fetch_sub(&A->ValDoNotUse, V, MO); +} + +template <typename T> +INLINE typename T::Type atomic_exchange(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + typename T::Type R; + __atomic_exchange(&A->ValDoNotUse, &V, &R, MO); + return R; +} + +template <typename T> +INLINE bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, + typename T::Type Xchg, + memory_order MO) { + return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, false, MO, + __ATOMIC_RELAXED); +} + +template <typename T> +INLINE bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp, + typename T::Type Xchg, + memory_order MO) { + return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, true, MO, + __ATOMIC_RELAXED); +} + +// Clutter-reducing helpers. + +template <typename T> +INLINE typename T::Type atomic_load_relaxed(const volatile T *A) { + return atomic_load(A, memory_order_relaxed); +} + +template <typename T> +INLINE void atomic_store_relaxed(volatile T *A, typename T::Type V) { + atomic_store(A, V, memory_order_relaxed); +} + +} // namespace scudo + +#endif // SCUDO_ATOMIC_H_ diff --git a/lib/scudo/standalone/bytemap.h b/lib/scudo/standalone/bytemap.h new file mode 100644 index 000000000..2c8ba1fd0 --- /dev/null +++ b/lib/scudo/standalone/bytemap.h @@ -0,0 +1,103 @@ +//===-- bytemap.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_BYTEMAP_H_ +#define SCUDO_BYTEMAP_H_ + +#include "atomic_helpers.h" +#include "common.h" +#include "mutex.h" + +namespace scudo { + +template <uptr Size> class FlatByteMap { +public: + void initLinkerInitialized() { + Map = reinterpret_cast<u8 *>(map(nullptr, Size, "scudo:bytemap")); + } + void init() { initLinkerInitialized(); } + + void set(uptr Index, u8 Value) { + DCHECK_LT(Index, Size); + DCHECK_EQ(0U, Map[Index]); + Map[Index] = Value; + } + u8 operator[](uptr Index) { + DCHECK_LT(Index, Size); + return Map[Index]; + } + +private: + u8 *Map; +}; + +template <uptr Level1Size, uptr Level2Size> class TwoLevelByteMap { +public: + void initLinkerInitialized() { + Level1Map = reinterpret_cast<atomic_uptr *>( + map(nullptr, sizeof(atomic_uptr) * Level1Size, "scudo:bytemap")); + } + void init() { + initLinkerInitialized(); + Mutex.init(); + } + + void reset() { + for (uptr I = 0; I < Level1Size; I++) { + u8 *P = get(I); + if (!P) + continue; + unmap(P, Level2Size); + } + memset(Level1Map, 0, sizeof(atomic_uptr) * Level1Size); + } + + uptr size() const { return Level1Size * Level2Size; } + + void set(uptr Index, u8 Value) { + DCHECK_LT(Index, Level1Size * Level2Size); + u8 *Level2Map = getOrCreate(Index / Level2Size); + DCHECK_EQ(0U, Level2Map[Index % Level2Size]); + Level2Map[Index % Level2Size] = Value; + } + + u8 operator[](uptr Index) const { + DCHECK_LT(Index, Level1Size * Level2Size); + u8 *Level2Map = get(Index / Level2Size); + if (!Level2Map) + return 0; + return Level2Map[Index % Level2Size]; + } + +private: + u8 *get(uptr Index) const { + DCHECK_LT(Index, Level1Size); + return reinterpret_cast<u8 *>( + atomic_load(&Level1Map[Index], memory_order_acquire)); + } + + u8 *getOrCreate(uptr Index) { + u8 *Res = get(Index); + if (!Res) { + SpinMutexLock L(&Mutex); + if (!(Res = get(Index))) { + Res = reinterpret_cast<u8 *>(map(nullptr, Level2Size, "scudo:bytemap")); + atomic_store(&Level1Map[Index], reinterpret_cast<uptr>(Res), + memory_order_release); + } + } + return Res; + } + + atomic_uptr *Level1Map; + StaticSpinMutex Mutex; +}; + +} // namespace scudo + +#endif // SCUDO_BYTEMAP_H_ diff --git a/lib/scudo/standalone/checksum.cc b/lib/scudo/standalone/checksum.cc new file mode 100644 index 000000000..ff6462bcd --- /dev/null +++ b/lib/scudo/standalone/checksum.cc @@ -0,0 +1,70 @@ +//===-- checksum.cc ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "checksum.h" +#include "atomic_helpers.h" + +#if defined(__x86_64__) || defined(__i386__) +#include <cpuid.h> +#elif defined(__arm__) || defined(__aarch64__) +#if SCUDO_FUCHSIA +#include <zircon/features.h> +#include <zircon/syscalls.h> +#else +#include <sys/auxv.h> +#endif +#endif + +namespace scudo { + +atomic_u8 HashAlgorithm = {BSDChecksum}; + +#if defined(__x86_64__) || defined(__i386__) +// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. +// CRC32 requires the SSE 4.2 instruction set. +#ifndef bit_SSE4_2 +#define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. +#endif + +bool hasHardwareCRC32() { + u32 Eax, Ebx = 0, Ecx = 0, Edx = 0; + __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); + const bool IsIntel = (Ebx == signature_INTEL_ebx) && + (Edx == signature_INTEL_edx) && + (Ecx == signature_INTEL_ecx); + const bool IsAMD = (Ebx == signature_AMD_ebx) && (Edx == signature_AMD_edx) && + (Ecx == signature_AMD_ecx); + if (!IsIntel && !IsAMD) + return false; + __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); + return !!(Ecx & bit_SSE4_2); +} + +#elif defined(__arm__) || defined(__aarch64__) +#ifndef AT_HWCAP +#define AT_HWCAP 16 +#endif +#ifndef HWCAP_CRC32 +#define HWCAP_CRC32 (1U << 7) // HWCAP_CRC32 is missing on older platforms. +#endif + +bool hasHardwareCRC32() { +#if SCUDO_FUCHSIA + u32 HWCap; + const zx_status_t Status = + zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap); + if (Status != ZX_OK) + return false; + return !!(HWCap & ZX_ARM64_FEATURE_ISA_CRC32); +#else + return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); +#endif // SCUDO_FUCHSIA +} +#endif // defined(__x86_64__) || defined(__i386__) + +} // namespace scudo diff --git a/lib/scudo/standalone/checksum.h b/lib/scudo/standalone/checksum.h new file mode 100644 index 000000000..7c4afcd96 --- /dev/null +++ b/lib/scudo/standalone/checksum.h @@ -0,0 +1,54 @@ +//===-- checksum.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CHECKSUM_H_ +#define SCUDO_CHECKSUM_H_ + +#include "internal_defs.h" + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -msse4.2 +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. + +#ifdef __SSE4_2__ +#include <smmintrin.h> +#define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +#endif +#ifdef __ARM_FEATURE_CRC32 +#include <arm_acle.h> +#define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +#endif + +namespace scudo { + +enum ChecksumType : u8 { + BSDChecksum = 0, + HardwareCRC32 = 1, +}; + +// BSD checksum, unlike a software CRC32, doesn't use any array lookup. We save +// significantly on memory accesses, as well as 1K of CRC32 table, on platforms +// that do no support hardware CRC32. The checksum itself is 16-bit, which is at +// odds with CRC32, but enough for our needs. +INLINE u16 computeBSDChecksum(u16 Sum, uptr Data) { + for (u8 I = 0; I < sizeof(Data); I++) { + Sum = static_cast<u16>((Sum >> 1) | ((Sum & 1) << 15)); + Sum = static_cast<u16>(Sum + (Data & 0xff)); + Data >>= 8; + } + return Sum; +} + +bool hasHardwareCRC32(); +WEAK u32 computeHardwareCRC32(u32 Crc, uptr Data); + +} // namespace scudo + +#endif // SCUDO_CHECKSUM_H_ diff --git a/lib/scudo/standalone/common.cc b/lib/scudo/standalone/common.cc new file mode 100644 index 000000000..2a26efbb9 --- /dev/null +++ b/lib/scudo/standalone/common.cc @@ -0,0 +1,32 @@ +//===-- common.cc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.h" +#include "atomic_helpers.h" + +namespace scudo { + +uptr PageSizeCached; +uptr getPageSize(); + +uptr getPageSizeSlow() { + PageSizeCached = getPageSize(); + CHECK_NE(PageSizeCached, 0); + return PageSizeCached; +} + +// Fatal internal map() or unmap() error (potentially OOM related). +void NORETURN dieOnMapUnmapError(bool OutOfMemory) { + outputRaw("Scudo ERROR: internal map or unmap failure"); + if (OutOfMemory) + outputRaw(" (OOM)"); + outputRaw("\n"); + die(); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/common.h b/lib/scudo/standalone/common.h new file mode 100644 index 000000000..313f89c9c --- /dev/null +++ b/lib/scudo/standalone/common.h @@ -0,0 +1,175 @@ +//===-- common.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_COMMON_H_ +#define SCUDO_COMMON_H_ + +#include "internal_defs.h" + +#include "fuchsia.h" +#include "linux.h" + +#include <stddef.h> +#include <string.h> + +namespace scudo { + +template <class Dest, class Source> INLINE Dest bit_cast(const Source &S) { + COMPILER_CHECK(sizeof(Dest) == sizeof(Source)); + Dest D; + memcpy(&D, &S, sizeof(D)); + return D; +} + +INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) { + return (X + Boundary - 1) & ~(Boundary - 1); +} + +INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) { + return X & ~(Boundary - 1); +} + +INLINE constexpr bool isAligned(uptr X, uptr Alignment) { + return (X & (Alignment - 1)) == 0; +} + +template <class T> constexpr T Min(T A, T B) { return A < B ? A : B; } + +template <class T> constexpr T Max(T A, T B) { return A > B ? A : B; } + +template <class T> void Swap(T &A, T &B) { + T Tmp = A; + A = B; + B = Tmp; +} + +INLINE bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } + +INLINE uptr getMostSignificantSetBitIndex(uptr X) { + DCHECK_NE(X, 0U); + return SCUDO_WORDSIZE - 1U - static_cast<uptr>(__builtin_clzl(X)); +} + +INLINE uptr roundUpToPowerOfTwo(uptr Size) { + DCHECK(Size); + if (isPowerOfTwo(Size)) + return Size; + const uptr Up = getMostSignificantSetBitIndex(Size); + DCHECK_LT(Size, (1UL << (Up + 1))); + DCHECK_GT(Size, (1UL << Up)); + return 1UL << (Up + 1); +} + +INLINE uptr getLeastSignificantSetBitIndex(uptr X) { + DCHECK_NE(X, 0U); + return static_cast<uptr>(__builtin_ctzl(X)); +} + +INLINE uptr getLog2(uptr X) { + DCHECK(isPowerOfTwo(X)); + return getLeastSignificantSetBitIndex(X); +} + +INLINE u32 getRandomU32(u32 *State) { + // ANSI C linear congruential PRNG (16-bit output). + // return (*State = *State * 1103515245 + 12345) >> 16; + // XorShift (32-bit output). + *State ^= *State << 13; + *State ^= *State >> 17; + *State ^= *State << 5; + return *State; +} + +INLINE u32 getRandomModN(u32 *State, u32 N) { + return getRandomU32(State) % N; // [0, N) +} + +template <typename T> INLINE void shuffle(T *A, u32 N, u32 *RandState) { + if (N <= 1) + return; + u32 State = *RandState; + for (u32 I = N - 1; I > 0; I--) + Swap(A[I], A[getRandomModN(&State, I + 1)]); + *RandState = State; +} + +// Hardware specific inlinable functions. + +INLINE void yieldProcessor(u8 Count) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("" ::: "memory"); + for (u8 I = 0; I < Count; I++) + __asm__ __volatile__("pause"); +#elif defined(__aarch64__) || defined(__arm__) + __asm__ __volatile__("" ::: "memory"); + for (u8 I = 0; I < Count; I++) + __asm__ __volatile__("yield"); +#endif + __asm__ __volatile__("" ::: "memory"); +} + +// Platform specific functions. + +void yieldPlatform(); + +extern uptr PageSizeCached; +uptr getPageSizeSlow(); +INLINE uptr getPageSizeCached() { + if (LIKELY(PageSizeCached)) + return PageSizeCached; + return getPageSizeSlow(); +} + +u32 getNumberOfCPUs(); + +const char *getEnv(const char *Name); + +u64 getMonotonicTime(); + +// Our randomness gathering function is limited to 256 bytes to ensure we get +// as many bytes as requested, and avoid interruptions (on Linux). +constexpr uptr MaxRandomLength = 256U; +bool getRandom(void *Buffer, uptr Length, bool Blocking = false); + +// Platform memory mapping functions. + +#define MAP_ALLOWNOMEM (1U << 0) +#define MAP_NOACCESS (1U << 1) +#define MAP_RESIZABLE (1U << 2) + +// Our platform memory mapping use is restricted to 3 scenarios: +// - reserve memory at a random address (MAP_NOACCESS); +// - commit memory in a previously reserved space; +// - commit memory at a random address. +// As such, only a subset of parameters combinations is valid, which is checked +// by the function implementation. The Data parameter allows to pass opaque +// platform specific data to the function. +// Returns nullptr on error or dies if MAP_ALLOWNOMEM is not specified. +void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0, + MapPlatformData *Data = nullptr); + +// Indicates that we are getting rid of the whole mapping, which might have +// further consequences on Data, depending on the platform. +#define UNMAP_ALL (1U << 0) + +void unmap(void *Addr, uptr Size, uptr Flags = 0, + MapPlatformData *Data = nullptr); + +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, + MapPlatformData *Data = nullptr); + +// Internal map & unmap fatal error. This must not call map(). +void NORETURN dieOnMapUnmapError(bool OutOfMemory = false); + +// Logging related functions. + +void setAbortMessage(const char *Message); + +} // namespace scudo + +#endif // SCUDO_COMMON_H_ diff --git a/lib/scudo/standalone/crc32_hw.cc b/lib/scudo/standalone/crc32_hw.cc new file mode 100644 index 000000000..f4dae7b5f --- /dev/null +++ b/lib/scudo/standalone/crc32_hw.cc @@ -0,0 +1,19 @@ +//===-- crc32_hw.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "checksum.h" + +namespace scudo { + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return static_cast<u32>(CRC32_INTRINSIC(Crc, Data)); +} +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace scudo diff --git a/lib/scudo/standalone/flags.cc b/lib/scudo/standalone/flags.cc new file mode 100644 index 000000000..21144f211 --- /dev/null +++ b/lib/scudo/standalone/flags.cc @@ -0,0 +1,57 @@ +//===-- flags.cc ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "flags.h" +#include "common.h" +#include "flags_parser.h" +#include "interface.h" + +namespace scudo { + +Flags *getFlags() { + static Flags F; + return &F; +} + +void Flags::setDefaults() { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "flags.inc" +#undef SCUDO_FLAG +} + +void registerFlags(FlagParser *Parser, Flags *F) { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) \ + Parser->registerFlag(#Name, Description, FlagType::FT_##Type, \ + reinterpret_cast<void *>(&F->Name)); +#include "flags.inc" +#undef SCUDO_FLAG +} + +static const char *getCompileDefinitionScudoDefaultOptions() { +#ifdef SCUDO_DEFAULT_OPTIONS + return STRINGIFY(SCUDO_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +static const char *getScudoDefaultOptions() { + return (&__scudo_default_options) ? __scudo_default_options() : ""; +} + +void initFlags() { + Flags *F = getFlags(); + F->setDefaults(); + FlagParser Parser; + registerFlags(&Parser, F); + Parser.parseString(getCompileDefinitionScudoDefaultOptions()); + Parser.parseString(getScudoDefaultOptions()); + Parser.parseString(getEnv("SCUDO_OPTIONS")); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/flags.h b/lib/scudo/standalone/flags.h new file mode 100644 index 000000000..edd39a1b8 --- /dev/null +++ b/lib/scudo/standalone/flags.h @@ -0,0 +1,30 @@ +//===-- flags.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAGS_H_ +#define SCUDO_FLAGS_H_ + +#include "internal_defs.h" + +namespace scudo { + +struct Flags { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "flags.inc" +#undef SCUDO_FLAG + void setDefaults(); +}; + +Flags *getFlags(); +void initFlags(); +class FlagParser; +void registerFlags(FlagParser *Parser, Flags *F); + +} // namespace scudo + +#endif // SCUDO_FLAGS_H_ diff --git a/lib/scudo/standalone/flags.inc b/lib/scudo/standalone/flags.inc new file mode 100644 index 000000000..25b86e14f --- /dev/null +++ b/lib/scudo/standalone/flags.inc @@ -0,0 +1,50 @@ +//===-- flags.inc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAG +#error "Define SCUDO_FLAG prior to including this file!" +#endif + +SCUDO_FLAG(int, quarantine_size_kb, 0, + "Size (in kilobytes) of quarantine used to delay the actual " + "deallocation of chunks. Lower value may reduce memory usage but " + "decrease the effectiveness of the mitigation.") + +SCUDO_FLAG(int, thread_local_quarantine_size_kb, 0, + "Size (in kilobytes) of per-thread cache used to offload the global " + "quarantine. Lower value may reduce memory usage but might increase " + "the contention on the global quarantine.") + +SCUDO_FLAG(int, quarantine_max_chunk_size, 0, + "Size (in bytes) up to which chunks will be quarantined (if lower " + "than or equal to).") + +SCUDO_FLAG(bool, dealloc_type_mismatch, false, + "Terminate on a type mismatch in allocation-deallocation functions, " + "eg: malloc/delete, new/free, new/delete[], etc.") + +SCUDO_FLAG(bool, delete_size_mismatch, true, + "Terminate on a size mismatch between a sized-delete and the actual " + "size of a chunk (as provided to new/new[]).") + +SCUDO_FLAG(bool, zero_contents, false, "Zero chunk contents on allocation.") + +SCUDO_FLAG(int, rss_limit_mb, -1, + "Enforce an upper limit (in megabytes) to the process RSS. The " + "allocator will terminate or return NULL when allocations are " + "attempted past that limit (depending on may_return_null). Negative " + "values disable the feature.") + +SCUDO_FLAG(bool, may_return_null, true, + "Indicate whether the allocator should terminate instead of " + "returning NULL in otherwise non-fatal error scenarios, eg: OOM, " + "invalid allocation alignments, etc.") + +SCUDO_FLAG(int, release_to_os_interval_ms, 5000, + "Interval (in milliseconds) at which to attempt release of unused " + "memory to the OS. Negative values disable the feature.") diff --git a/lib/scudo/standalone/flags_parser.cc b/lib/scudo/standalone/flags_parser.cc new file mode 100644 index 000000000..918304f49 --- /dev/null +++ b/lib/scudo/standalone/flags_parser.cc @@ -0,0 +1,163 @@ +//===-- flags_parser.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "flags_parser.h" +#include "common.h" +#include "report.h" + +#include <string.h> + +namespace scudo { + +class UnknownFlagsRegistry { + static const u32 MaxUnknownFlags = 16; + const char *UnknownFlagsNames[MaxUnknownFlags]; + u32 NumberOfUnknownFlags; + +public: + void add(const char *Name) { + CHECK_LT(NumberOfUnknownFlags, MaxUnknownFlags); + UnknownFlagsNames[NumberOfUnknownFlags++] = Name; + } + + void report() { + if (!NumberOfUnknownFlags) + return; + Printf("Scudo WARNING: found %d unrecognized flag(s):\n", + NumberOfUnknownFlags); + for (u32 I = 0; I < NumberOfUnknownFlags; ++I) + Printf(" %s\n", UnknownFlagsNames[I]); + NumberOfUnknownFlags = 0; + } +}; +static UnknownFlagsRegistry UnknownFlags; + +void reportUnrecognizedFlags() { UnknownFlags.report(); } + +void FlagParser::printFlagDescriptions() { + Printf("Available flags for Scudo:\n"); + for (u32 I = 0; I < NumberOfFlags; ++I) + Printf("\t%s\n\t\t- %s\n", Flags[I].Name, Flags[I].Desc); +} + +static bool isSeparator(char C) { + return C == ' ' || C == ',' || C == ':' || C == '\n' || C == '\t' || + C == '\r'; +} + +static bool isSeparatorOrNull(char C) { return !C || isSeparator(C); } + +void FlagParser::skipWhitespace() { + while (isSeparator(Buffer[Pos])) + ++Pos; +} + +void FlagParser::parseFlag() { + const uptr NameStart = Pos; + while (Buffer[Pos] != '=' && !isSeparatorOrNull(Buffer[Pos])) + ++Pos; + if (Buffer[Pos] != '=') + reportError("expected '='"); + const char *Name = Buffer + NameStart; + const uptr ValueStart = ++Pos; + const char *Value; + if (Buffer[Pos] == '\'' || Buffer[Pos] == '"') { + const char Quote = Buffer[Pos++]; + while (Buffer[Pos] != 0 && Buffer[Pos] != Quote) + ++Pos; + if (Buffer[Pos] == 0) + reportError("unterminated string"); + Value = Buffer + ValueStart + 1; + ++Pos; // consume the closing quote + } else { + while (!isSeparatorOrNull(Buffer[Pos])) + ++Pos; + Value = Buffer + ValueStart; + } + if (!runHandler(Name, Value)) + reportError("flag parsing failed."); +} + +void FlagParser::parseFlags() { + while (true) { + skipWhitespace(); + if (Buffer[Pos] == 0) + break; + parseFlag(); + } +} + +void FlagParser::parseString(const char *S) { + if (!S) + return; + // Backup current parser state to allow nested parseString() calls. + const char *OldBuffer = Buffer; + const uptr OldPos = Pos; + Buffer = S; + Pos = 0; + + parseFlags(); + + Buffer = OldBuffer; + Pos = OldPos; +} + +INLINE bool parseBool(const char *Value, bool *b) { + if (strncmp(Value, "0", 1) == 0 || strncmp(Value, "no", 2) == 0 || + strncmp(Value, "false", 5) == 0) { + *b = false; + return true; + } + if (strncmp(Value, "1", 1) == 0 || strncmp(Value, "yes", 3) == 0 || + strncmp(Value, "true", 4) == 0) { + *b = true; + return true; + } + return false; +} + +bool FlagParser::runHandler(const char *Name, const char *Value) { + for (u32 I = 0; I < NumberOfFlags; ++I) { + const uptr Len = strlen(Flags[I].Name); + if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != '=') + continue; + bool Ok = false; + switch (Flags[I].Type) { + case FlagType::FT_bool: + Ok = parseBool(Value, reinterpret_cast<bool *>(Flags[I].Var)); + if (!Ok) + reportInvalidFlag("bool", Value); + break; + case FlagType::FT_int: + char *ValueEnd; + *reinterpret_cast<int *>(Flags[I].Var) = + static_cast<int>(strtol(Value, &ValueEnd, 10)); + Ok = + *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd); + if (!Ok) + reportInvalidFlag("int", Value); + break; + } + return Ok; + } + // Unrecognized flag. This is not a fatal error, we may print a warning later. + UnknownFlags.add(Name); + return true; +} + +void FlagParser::registerFlag(const char *Name, const char *Desc, FlagType Type, + void *Var) { + CHECK_LT(NumberOfFlags, MaxFlags); + Flags[NumberOfFlags].Name = Name; + Flags[NumberOfFlags].Desc = Desc; + Flags[NumberOfFlags].Type = Type; + Flags[NumberOfFlags].Var = Var; + ++NumberOfFlags; +} + +} // namespace scudo diff --git a/lib/scudo/standalone/flags_parser.h b/lib/scudo/standalone/flags_parser.h new file mode 100644 index 000000000..d65bff7e7 --- /dev/null +++ b/lib/scudo/standalone/flags_parser.h @@ -0,0 +1,56 @@ +//===-- flags_parser.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAGS_PARSER_H_ +#define SCUDO_FLAGS_PARSER_H_ + +#include "report.h" +#include "string_utils.h" + +#include <stddef.h> +#include <stdlib.h> + +namespace scudo { + +enum class FlagType : u8 { + FT_bool, + FT_int, +}; + +class FlagParser { +public: + void registerFlag(const char *Name, const char *Desc, FlagType Type, + void *Var); + void parseString(const char *S); + void printFlagDescriptions(); + +private: + static const u32 MaxFlags = 12; + struct Flag { + const char *Name; + const char *Desc; + FlagType Type; + void *Var; + } Flags[MaxFlags]; + + u32 NumberOfFlags = 0; + const char *Buffer = nullptr; + uptr Pos = 0; + + void reportFatalError(const char *Error); + void skipWhitespace(); + void parseFlags(); + void parseFlag(); + bool runHandler(const char *Name, const char *Value); +}; + +void reportUnrecognizedFlags(); + +} // namespace scudo + +#endif // SCUDO_FLAGS_PARSER_H_ diff --git a/lib/scudo/standalone/fuchsia.cc b/lib/scudo/standalone/fuchsia.cc new file mode 100644 index 000000000..e54563138 --- /dev/null +++ b/lib/scudo/standalone/fuchsia.cc @@ -0,0 +1,186 @@ +//===-- fuchsia.cc ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_FUCHSIA + +#include "common.h" +#include "mutex.h" +#include "string_utils.h" + +#include <limits.h> // for PAGE_SIZE +#include <stdlib.h> // for getenv() +#include <zircon/sanitizer.h> +#include <zircon/syscalls.h> + +namespace scudo { + +void yieldPlatform() { + const zx_status_t Status = _zx_nanosleep(0); + CHECK_EQ(Status, ZX_OK); +} + +uptr getPageSize() { return PAGE_SIZE; } + +void NORETURN die() { __builtin_trap(); } + +// We zero-initialize the Extra parameter of map(), make sure this is consistent +// with ZX_HANDLE_INVALID. +COMPILER_CHECK(ZX_HANDLE_INVALID == 0); + +static void *allocateVmar(uptr Size, MapPlatformData *Data, bool AllowNoMem) { + // Only scenario so far. + DCHECK(Data); + DCHECK_EQ(Data->Vmar, ZX_HANDLE_INVALID); + + const zx_status_t Status = _zx_vmar_allocate( + _zx_vmar_root_self(), + ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0, + Size, &Data->Vmar, &Data->VmarBase); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + return reinterpret_cast<void *>(Data->VmarBase); +} + +void *map(void *Addr, uptr Size, const char *Name, uptr Flags, + MapPlatformData *Data) { + DCHECK_EQ(Size % PAGE_SIZE, 0); + const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM); + + // For MAP_NOACCESS, just allocate a Vmar and return. + if (Flags & MAP_NOACCESS) + return allocateVmar(Size, Data, AllowNoMem); + + const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self(); + CHECK_NE(Vmar, ZX_HANDLE_INVALID); + + zx_status_t Status; + zx_handle_t Vmo; + uint64_t VmoSize = 0; + if (Data && Data->Vmo != ZX_HANDLE_INVALID) { + // If a Vmo was specified, it's a resize operation. + CHECK(Addr); + DCHECK(Flags & MAP_RESIZABLE); + Vmo = Data->Vmo; + VmoSize = Data->VmoSize; + Status = _zx_vmo_set_size(Vmo, VmoSize + Size); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + } else { + // Otherwise, create a Vmo and set its name. + Status = _zx_vmo_create(Size, ZX_VMO_RESIZABLE, &Vmo); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + _zx_object_set_property(Vmo, ZX_PROP_NAME, Name, strlen(Name)); + } + + uintptr_t P; + zx_vm_option_t MapFlags = ZX_VM_PERM_READ | ZX_VM_PERM_WRITE; + const uint64_t Offset = + Addr ? reinterpret_cast<uintptr_t>(Addr) - Data->VmarBase : 0; + if (Offset) + MapFlags |= ZX_VM_SPECIFIC; + Status = _zx_vmar_map(Vmar, MapFlags, Offset, Vmo, VmoSize, Size, &P); + // No need to track the Vmo if we don't intend on resizing it. Close it. + if (Flags & MAP_RESIZABLE) { + DCHECK(Data); + DCHECK_EQ(Data->Vmo, ZX_HANDLE_INVALID); + Data->Vmo = Vmo; + } else { + CHECK_EQ(_zx_handle_close(Vmo), ZX_OK); + } + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + if (Data) + Data->VmoSize += Size; + + return reinterpret_cast<void *>(P); +} + +void unmap(void *Addr, uptr Size, uptr Flags, MapPlatformData *Data) { + if (Flags & UNMAP_ALL) { + DCHECK_NE(Data, nullptr); + const zx_handle_t Vmar = Data->Vmar; + DCHECK_NE(Vmar, _zx_vmar_root_self()); + // Destroying the vmar effectively unmaps the whole mapping. + CHECK_EQ(_zx_vmar_destroy(Vmar), ZX_OK); + CHECK_EQ(_zx_handle_close(Vmar), ZX_OK); + } else { + const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self(); + const zx_status_t Status = + _zx_vmar_unmap(Vmar, reinterpret_cast<uintptr_t>(Addr), Size); + if (Status != ZX_OK) + dieOnMapUnmapError(); + } + if (Data) { + if (Data->Vmo != ZX_HANDLE_INVALID) + CHECK_EQ(_zx_handle_close(Data->Vmo), ZX_OK); + memset(Data, 0, sizeof(*Data)); + } +} + +void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size, + MapPlatformData *Data) { + DCHECK(Data); + DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID); + DCHECK_NE(Data->Vmo, ZX_HANDLE_INVALID); + const zx_status_t Status = + _zx_vmo_op_range(Data->Vmo, ZX_VMO_OP_DECOMMIT, Offset, Size, NULL, 0); + CHECK_EQ(Status, ZX_OK); +} + +const char *getEnv(const char *Name) { return getenv(Name); } + +void BlockingMutex::wait() { + const zx_status_t Status = + _zx_futex_wait(reinterpret_cast<zx_futex_t *>(OpaqueStorage), MtxSleeping, + ZX_HANDLE_INVALID, ZX_TIME_INFINITE); + if (Status != ZX_ERR_BAD_STATE) + CHECK_EQ(Status, ZX_OK); // Normal race +} + +void BlockingMutex::wake() { + const zx_status_t Status = + _zx_futex_wake(reinterpret_cast<zx_futex_t *>(OpaqueStorage), 1); + CHECK_EQ(Status, ZX_OK); +} + +u64 getMonotonicTime() { return _zx_clock_get_monotonic(); } + +u32 getNumberOfCPUs() { return _zx_system_get_num_cpus(); } + +bool getRandom(void *Buffer, uptr Length, bool Blocking) { + COMPILER_CHECK(MaxRandomLength <= ZX_CPRNG_DRAW_MAX_LEN); + if (!Buffer || !Length || Length > MaxRandomLength) + return false; + _zx_cprng_draw(Buffer, Length); + return true; +} + +void outputRaw(const char *Buffer) { + __sanitizer_log_write(Buffer, strlen(Buffer)); +} + +void setAbortMessage(const char *Message) {} + +} // namespace scudo + +#endif // SCUDO_FUCHSIA diff --git a/lib/scudo/standalone/fuchsia.h b/lib/scudo/standalone/fuchsia.h new file mode 100644 index 000000000..d6993f892 --- /dev/null +++ b/lib/scudo/standalone/fuchsia.h @@ -0,0 +1,31 @@ +//===-- fuchsia.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FUCHSIA_H_ +#define SCUDO_FUCHSIA_H_ + +#include "platform.h" + +#if SCUDO_FUCHSIA + +#include <zircon/process.h> + +namespace scudo { + +struct MapPlatformData { + zx_handle_t Vmar; + zx_handle_t Vmo; + uintptr_t VmarBase; + uint64_t VmoSize; +}; + +} // namespace scudo + +#endif // SCUDO_FUCHSIA + +#endif // SCUDO_FUCHSIA_H_ diff --git a/lib/scudo/standalone/interface.h b/lib/scudo/standalone/interface.h new file mode 100644 index 000000000..e2639823f --- /dev/null +++ b/lib/scudo/standalone/interface.h @@ -0,0 +1,29 @@ +//===-- interface.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERFACE_H_ +#define SCUDO_INTERFACE_H_ + +#include "internal_defs.h" + +extern "C" { + +WEAK INTERFACE const char *__scudo_default_options(); + +// Post-allocation & pre-deallocation hooks. +// They must be thread-safe and not use heap related functions. +WEAK INTERFACE void __scudo_allocate_hook(void *ptr, size_t size); +WEAK INTERFACE void __scudo_deallocate_hook(void *ptr); + +WEAK INTERFACE void __scudo_print_stats(void); + +typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); + +} // extern "C" + +#endif // SCUDO_INTERFACE_H_ diff --git a/lib/scudo/standalone/internal_defs.h b/lib/scudo/standalone/internal_defs.h new file mode 100644 index 000000000..cca9c42af --- /dev/null +++ b/lib/scudo/standalone/internal_defs.h @@ -0,0 +1,135 @@ +//===-- internal_defs.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERNAL_DEFS_H_ +#define SCUDO_INTERNAL_DEFS_H_ + +#include "platform.h" + +#include <stdint.h> + +#ifndef SCUDO_DEBUG +#define SCUDO_DEBUG 0 +#endif + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + +// String related macros. + +#define STRINGIFY_(S) #S +#define STRINGIFY(S) STRINGIFY_(S) +#define CONCATENATE_(S, C) S##C +#define CONCATENATE(S, C) CONCATENATE_(S, C) + +// Attributes & builtins related macros. + +#define INTERFACE __attribute__((visibility("default"))) +#define WEAK __attribute__((weak)) +#define INLINE inline +#define ALWAYS_INLINE inline __attribute__((always_inline)) +#define ALIAS(X) __attribute__((alias(X))) +// Please only use the ALIGNED macro before the type. Using ALIGNED after the +// variable declaration is not portable. +#define ALIGNED(X) __attribute__((aligned(X))) +#define FORMAT(F, A) __attribute__((format(printf, F, A))) +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#define THREADLOCAL __thread +#define LIKELY(X) __builtin_expect(!!(X), 1) +#define UNLIKELY(X) __builtin_expect(!!(X), 0) +#if defined(__i386__) || defined(__x86_64__) +// __builtin_prefetch(X) generates prefetchnt0 on x86 +#define PREFETCH(X) __asm__("prefetchnta (%0)" : : "r"(X)) +#else +#define PREFETCH(X) __builtin_prefetch(X) +#endif +#define UNUSED __attribute__((unused)) +#define USED __attribute__((used)) +#define NOEXCEPT noexcept + +namespace scudo { + +typedef unsigned long uptr; +typedef signed long sptr; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; +typedef signed char s8; +typedef signed short s16; +typedef signed int s32; +typedef signed long long s64; + +// The following two functions have platform specific implementations. +void outputRaw(const char *Buffer); +void NORETURN die(); + +#define RAW_CHECK_MSG(Expr, Msg) \ + do { \ + if (UNLIKELY(!(Expr))) { \ + outputRaw(Msg); \ + die(); \ + } \ + } while (false) + +#define RAW_CHECK(Expr) RAW_CHECK_MSG(Expr, #Expr) + +void NORETURN reportCheckFailed(const char *File, int Line, + const char *Condition, u64 Value1, u64 Value2); + +#define CHECK_IMPL(C1, Op, C2) \ + do { \ + u64 V1 = (u64)(C1); \ + u64 V2 = (u64)(C2); \ + if (UNLIKELY(!(V1 Op V2))) { \ + reportCheckFailed(__FILE__, __LINE__, "(" #C1 ") " #Op " (" #C2 ")", V1, \ + V2); \ + die(); \ + } \ + } while (false) + +#define CHECK(A) CHECK_IMPL((A), !=, 0) +#define CHECK_EQ(A, B) CHECK_IMPL((A), ==, (B)) +#define CHECK_NE(A, B) CHECK_IMPL((A), !=, (B)) +#define CHECK_LT(A, B) CHECK_IMPL((A), <, (B)) +#define CHECK_LE(A, B) CHECK_IMPL((A), <=, (B)) +#define CHECK_GT(A, B) CHECK_IMPL((A), >, (B)) +#define CHECK_GE(A, B) CHECK_IMPL((A), >=, (B)) + +#if SCUDO_DEBUG +#define DCHECK(A) CHECK(A) +#define DCHECK_EQ(A, B) CHECK_EQ(A, B) +#define DCHECK_NE(A, B) CHECK_NE(A, B) +#define DCHECK_LT(A, B) CHECK_LT(A, B) +#define DCHECK_LE(A, B) CHECK_LE(A, B) +#define DCHECK_GT(A, B) CHECK_GT(A, B) +#define DCHECK_GE(A, B) CHECK_GE(A, B) +#else +#define DCHECK(A) +#define DCHECK_EQ(A, B) +#define DCHECK_NE(A, B) +#define DCHECK_LT(A, B) +#define DCHECK_LE(A, B) +#define DCHECK_GT(A, B) +#define DCHECK_GE(A, B) +#endif + +// The superfluous die() call effectively makes this macro NORETURN. +#define UNREACHABLE(Msg) \ + do { \ + CHECK(0 && Msg); \ + die(); \ + } while (0) + +#define COMPILER_CHECK(Pred) static_assert(Pred, "") + +enum LinkerInitialized { LINKER_INITIALIZED = 0 }; + +} // namespace scudo + +#endif // SCUDO_INTERNAL_DEFS_H_ diff --git a/lib/scudo/standalone/linux.cc b/lib/scudo/standalone/linux.cc new file mode 100644 index 000000000..5e695d72f --- /dev/null +++ b/lib/scudo/standalone/linux.cc @@ -0,0 +1,150 @@ +//===-- linux.cc ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "common.h" +#include "linux.h" +#include "mutex.h" +#include "string_utils.h" + +#include <errno.h> +#include <fcntl.h> +#include <linux/futex.h> +#include <sched.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <time.h> +#include <unistd.h> + +#if SCUDO_ANDROID +#include <sys/prctl.h> +// Definitions of prctl arguments to set a vma name in Android kernels. +#define ANDROID_PR_SET_VMA 0x53564d41 +#define ANDROID_PR_SET_VMA_ANON_NAME 0 +#endif + +namespace scudo { + +void yieldPlatform() { sched_yield(); } + +uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); } + +void NORETURN die() { abort(); } + +void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, + UNUSED MapPlatformData *Data) { + int MmapFlags = MAP_PRIVATE | MAP_ANON; + if (Flags & MAP_NOACCESS) + MmapFlags |= MAP_NORESERVE; + if (Addr) { + // Currently no scenario for a noaccess mapping with a fixed address. + DCHECK_EQ(Flags & MAP_NOACCESS, 0); + MmapFlags |= MAP_FIXED; + } + const int MmapProt = + (Flags & MAP_NOACCESS) ? PROT_NONE : PROT_READ | PROT_WRITE; + void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0); + if (P == MAP_FAILED) { + if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) + dieOnMapUnmapError(errno == ENOMEM); + return nullptr; + } +#if SCUDO_ANDROID + if (!(Flags & MAP_NOACCESS)) + prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name); +#endif + return P; +} + +void unmap(void *Addr, uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) { + if (munmap(Addr, Size) != 0) + dieOnMapUnmapError(); +} + +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, + UNUSED MapPlatformData *Data) { + void *Addr = reinterpret_cast<void *>(BaseAddress + Offset); + while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) { + } +} + +// Calling getenv should be fine (c)(tm) at any time. +const char *getEnv(const char *Name) { return getenv(Name); } + +void BlockingMutex::wait() { + syscall(SYS_futex, reinterpret_cast<uptr>(OpaqueStorage), FUTEX_WAIT_PRIVATE, + MtxSleeping, nullptr, nullptr, 0); +} + +void BlockingMutex::wake() { + syscall(SYS_futex, reinterpret_cast<uptr>(OpaqueStorage), FUTEX_WAKE_PRIVATE, + 1, nullptr, nullptr, 0); +} + +u64 getMonotonicTime() { + timespec TS; + clock_gettime(CLOCK_MONOTONIC, &TS); + return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) + + static_cast<u64>(TS.tv_nsec); +} + +u32 getNumberOfCPUs() { + cpu_set_t CPUs; + CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); + return static_cast<u32>(CPU_COUNT(&CPUs)); +} + +// Blocking is possibly unused if the getrandom block is not compiled in. +bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { + if (!Buffer || !Length || Length > MaxRandomLength) + return false; + ssize_t ReadBytes; +#if defined(SYS_getrandom) +#if !defined(GRND_NONBLOCK) +#define GRND_NONBLOCK 1 +#endif + // Up to 256 bytes, getrandom will not be interrupted. + ReadBytes = + syscall(SYS_getrandom, Buffer, Length, Blocking ? 0 : GRND_NONBLOCK); + if (ReadBytes == static_cast<ssize_t>(Length)) + return true; +#endif // defined(SYS_getrandom) + // Up to 256 bytes, a read off /dev/urandom will not be interrupted. + // Blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom. + const int FileDesc = open("/dev/urandom", O_RDONLY); + if (FileDesc == -1) + return false; + ReadBytes = read(FileDesc, Buffer, Length); + close(FileDesc); + return (ReadBytes == static_cast<ssize_t>(Length)); +} + +void outputRaw(const char *Buffer) { + static StaticSpinMutex Mutex; + SpinMutexLock L(&Mutex); + write(2, Buffer, strlen(Buffer)); +} + +extern "C" WEAK void android_set_abort_message(const char *); + +void setAbortMessage(const char *Message) { + if (&android_set_abort_message) + android_set_abort_message(Message); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/lib/scudo/standalone/linux.h b/lib/scudo/standalone/linux.h new file mode 100644 index 000000000..92c9eb5e9 --- /dev/null +++ b/lib/scudo/standalone/linux.h @@ -0,0 +1,70 @@ +//===-- linux.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_LINUX_H_ +#define SCUDO_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +namespace scudo { + +// MapPlatformData is unused on Linux, define it as a minimally sized structure. +struct MapPlatformData {}; + +#if SCUDO_ANDROID + +#if defined(__aarch64__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mrs %0, tpidr_el0" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__arm__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__i386__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("movl %%gs:0, %0" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__x86_64__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mov %%fs:0, %0" : "=r"(__v)); \ + __v; \ + }) +#else +#error "Unsupported architecture." +#endif + +// The Android Bionic team has allocated a TLS slot for sanitizers starting +// with Q, given that Android currently doesn't support ELF TLS. It is used to +// store sanitizer thread specific data. +static const int TLS_SLOT_SANITIZER = 8; // TODO(kostyak): 6 for Q!! + +ALWAYS_INLINE uptr *getAndroidTlsPtr() { + return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]); +} + +#endif // SCUDO_ANDROID + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_LINUX_H_ diff --git a/lib/scudo/standalone/list.h b/lib/scudo/standalone/list.h new file mode 100644 index 000000000..139e73eff --- /dev/null +++ b/lib/scudo/standalone/list.h @@ -0,0 +1,156 @@ +//===-- list.h --------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_LIST_H_ +#define SCUDO_LIST_H_ + +#include "internal_defs.h" + +namespace scudo { + +// Intrusive POD singly-linked list. +// An object with all zero fields should represent a valid empty list. clear() +// should be called on all non-zero-initialized objects before using. +template <class Item> struct IntrusiveList { + friend class Iterator; + + void clear() { + First = Last = nullptr; + Size = 0; + } + + bool empty() const { return Size == 0; } + uptr size() const { return Size; } + + void push_back(Item *X) { + if (empty()) { + X->Next = nullptr; + First = Last = X; + Size = 1; + } else { + X->Next = nullptr; + Last->Next = X; + Last = X; + Size++; + } + } + + void push_front(Item *X) { + if (empty()) { + X->Next = nullptr; + First = Last = X; + Size = 1; + } else { + X->Next = First; + First = X; + Size++; + } + } + + void pop_front() { + DCHECK(!empty()); + First = First->Next; + if (!First) + Last = nullptr; + Size--; + } + + void extract(Item *Prev, Item *X) { + DCHECK(!empty()); + DCHECK_NE(Prev, nullptr); + DCHECK_NE(X, nullptr); + DCHECK_EQ(Prev->Next, X); + Prev->Next = X->Next; + if (Last == X) + Last = Prev; + Size--; + } + + Item *front() { return First; } + const Item *front() const { return First; } + Item *back() { return Last; } + const Item *back() const { return Last; } + + void append_front(IntrusiveList<Item> *L) { + DCHECK_NE(this, L); + if (L->empty()) + return; + if (empty()) { + *this = *L; + } else if (!L->empty()) { + L->Last->Next = First; + First = L->First; + Size += L->size(); + } + L->clear(); + } + + void append_back(IntrusiveList<Item> *L) { + DCHECK_NE(this, L); + if (L->empty()) + return; + if (empty()) { + *this = *L; + } else { + Last->Next = L->First; + Last = L->Last; + Size += L->size(); + } + L->clear(); + } + + void checkConsistency() { + if (Size == 0) { + CHECK_EQ(First, 0); + CHECK_EQ(Last, 0); + } else { + uptr count = 0; + for (Item *I = First;; I = I->Next) { + count++; + if (I == Last) + break; + } + CHECK_EQ(size(), count); + CHECK_EQ(Last->Next, 0); + } + } + + template <class ItemT> class IteratorBase { + public: + explicit IteratorBase(ItemT *CurrentItem) : Current(CurrentItem) {} + IteratorBase &operator++() { + Current = Current->Next; + return *this; + } + bool operator!=(IteratorBase Other) const { + return Current != Other.Current; + } + ItemT &operator*() { return *Current; } + + private: + ItemT *Current; + }; + + typedef IteratorBase<Item> Iterator; + typedef IteratorBase<const Item> ConstIterator; + + Iterator begin() { return Iterator(First); } + Iterator end() { return Iterator(nullptr); } + + ConstIterator begin() const { return ConstIterator(First); } + ConstIterator end() const { return ConstIterator(nullptr); } + +private: + uptr Size; + Item *First; + Item *Last; +}; + +} // namespace scudo + +#endif // SCUDO_LIST_H_ diff --git a/lib/scudo/standalone/mutex.h b/lib/scudo/standalone/mutex.h new file mode 100644 index 000000000..6de3810b4 --- /dev/null +++ b/lib/scudo/standalone/mutex.h @@ -0,0 +1,108 @@ +//===-- mutex.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MUTEX_H_ +#define SCUDO_MUTEX_H_ + +#include "atomic_helpers.h" +#include "common.h" + +namespace scudo { + +class StaticSpinMutex { +public: + void init() { atomic_store_relaxed(&State, 0); } + + void lock() { + if (tryLock()) + return; + lockSlow(); + } + + bool tryLock() { + return atomic_exchange(&State, 1, memory_order_acquire) == 0; + } + + void unlock() { atomic_store(&State, 0, memory_order_release); } + + void checkLocked() { CHECK_EQ(atomic_load_relaxed(&State), 1); } + +private: + atomic_u8 State; + + void NOINLINE lockSlow() { + for (u32 I = 0;; I++) { + if (I < 10) + yieldProcessor(10); + else + yieldPlatform(); + if (atomic_load_relaxed(&State) == 0 && + atomic_exchange(&State, 1, memory_order_acquire) == 0) + return; + } + } +}; + +class SpinMutex : public StaticSpinMutex { +public: + SpinMutex() { init(); } + +private: + SpinMutex(const SpinMutex &) = delete; + void operator=(const SpinMutex &) = delete; +}; + +enum MutexState { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; + +class BlockingMutex { +public: + explicit constexpr BlockingMutex(LinkerInitialized) : OpaqueStorage{0} {} + BlockingMutex() { memset(this, 0, sizeof(*this)); } + void wait(); + void wake(); + void lock() { + atomic_u32 *M = reinterpret_cast<atomic_u32 *>(&OpaqueStorage); + if (atomic_exchange(M, MtxLocked, memory_order_acquire) == MtxUnlocked) + return; + while (atomic_exchange(M, MtxSleeping, memory_order_acquire) != MtxUnlocked) + wait(); + } + void unlock() { + atomic_u32 *M = reinterpret_cast<atomic_u32 *>(&OpaqueStorage); + const u32 V = atomic_exchange(M, MtxUnlocked, memory_order_release); + DCHECK_NE(V, MtxUnlocked); + if (V == MtxSleeping) + wake(); + } + void checkLocked() { + atomic_u32 *M = reinterpret_cast<atomic_u32 *>(&OpaqueStorage); + CHECK_NE(MtxUnlocked, atomic_load_relaxed(M)); + } + +private: + uptr OpaqueStorage[1]; +}; + +template <typename MutexType> class GenericScopedLock { +public: + explicit GenericScopedLock(MutexType *M) : Mutex(M) { Mutex->lock(); } + ~GenericScopedLock() { Mutex->unlock(); } + +private: + MutexType *Mutex; + + GenericScopedLock(const GenericScopedLock &) = delete; + void operator=(const GenericScopedLock &) = delete; +}; + +typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock; +typedef GenericScopedLock<BlockingMutex> BlockingMutexLock; + +} // namespace scudo + +#endif // SCUDO_MUTEX_H_ diff --git a/lib/scudo/standalone/platform.h b/lib/scudo/standalone/platform.h new file mode 100644 index 000000000..a897a566f --- /dev/null +++ b/lib/scudo/standalone/platform.h @@ -0,0 +1,70 @@ +//===-- platform.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#if defined(__linux__) +#define SCUDO_LINUX 1 +#else +#define SCUDO_LINUX 0 +#endif + +#if defined(__ANDROID__) +#define SCUDO_ANDROID 1 +#else +#define SCUDO_ANDROID 0 +#endif + +#if defined(__Fuchsia__) +#define SCUDO_FUCHSIA 1 +#else +#define SCUDO_FUCHSIA 0 +#endif + +#if __LP64__ +#define SCUDO_WORDSIZE 64U +#else +#define SCUDO_WORDSIZE 32U +#endif + +#if SCUDO_WORDSIZE == 64U +#define FIRST_32_SECOND_64(a, b) (b) +#else +#define FIRST_32_SECOND_64(a, b) (a) +#endif + +#ifndef SCUDO_CAN_USE_PRIMARY64 +#define SCUDO_CAN_USE_PRIMARY64 (SCUDO_WORDSIZE == 64U) +#endif + +#ifndef SCUDO_MIN_ALIGNMENT_LOG +// We force malloc-type functions to be aligned to std::max_align_t, but there +// is no reason why the minimum alignment for all other functions can't be 8 +// bytes. Except obviously for applications making incorrect assumptions. +// TODO(kostyak): define SCUDO_MIN_ALIGNMENT_LOG 3 +#define SCUDO_MIN_ALIGNMENT_LOG FIRST_32_SECOND_64(3, 4) +#endif + +#if defined(__aarch64__) +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +#else +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +#endif + +// Older gcc have issues aligning to a constexpr, and require an integer. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others. +#if defined(__powerpc__) || defined(__powerpc64__) +#define SCUDO_CACHE_LINE_SIZE 128 +#else +#define SCUDO_CACHE_LINE_SIZE 64 +#endif + +#define SCUDO_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) + +#endif // SCUDO_PLATFORM_H_ diff --git a/lib/scudo/standalone/release.h b/lib/scudo/standalone/release.h new file mode 100644 index 000000000..4fe29fde4 --- /dev/null +++ b/lib/scudo/standalone/release.h @@ -0,0 +1,262 @@ +//===-- release.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_RELEASE_H_ +#define SCUDO_RELEASE_H_ + +#include "common.h" +#include "list.h" + +namespace scudo { + +class ReleaseRecorder { +public: + ReleaseRecorder(uptr BaseAddress, MapPlatformData *Data = nullptr) + : BaseAddress(BaseAddress), Data(Data) {} + + uptr getReleasedRangesCount() const { return ReleasedRangesCount; } + + uptr getReleasedBytes() const { return ReleasedBytes; } + + // Releases [From, To) range of pages back to OS. + void releasePageRangeToOS(uptr From, uptr To) { + const uptr Size = To - From; + releasePagesToOS(BaseAddress, From, Size, Data); + ReleasedRangesCount++; + ReleasedBytes += Size; + } + +private: + uptr ReleasedRangesCount = 0; + uptr ReleasedBytes = 0; + uptr BaseAddress = 0; + MapPlatformData *Data = nullptr; +}; + +// A packed array of Counters. Each counter occupies 2^N bits, enough to store +// counter's MaxValue. Ctor will try to allocate the required Buffer via map() +// and the caller is expected to check whether the initialization was successful +// by checking isAllocated() result. For the performance sake, none of the +// accessors check the validity of the arguments, It is assumed that Index is +// always in [0, N) range and the value is not incremented past MaxValue. +class PackedCounterArray { +public: + PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) { + CHECK_GT(NumCounters, 0); + CHECK_GT(MaxValue, 0); + constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL; + // Rounding counter storage size up to the power of two allows for using + // bit shifts calculating particular counter's Index and offset. + const uptr CounterSizeBits = + roundUpToPowerOfTwo(getMostSignificantSetBitIndex(MaxValue) + 1); + CHECK_LE(CounterSizeBits, MaxCounterBits); + CounterSizeBitsLog = getLog2(CounterSizeBits); + CounterMask = ~(static_cast<uptr>(0)) >> (MaxCounterBits - CounterSizeBits); + + const uptr PackingRatio = MaxCounterBits >> CounterSizeBitsLog; + CHECK_GT(PackingRatio, 0); + PackingRatioLog = getLog2(PackingRatio); + BitOffsetMask = PackingRatio - 1; + + BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >> + PackingRatioLog) * + sizeof(*Buffer); + Buffer = reinterpret_cast<uptr *>( + map(nullptr, BufferSize, "scudo:counters", MAP_ALLOWNOMEM)); + } + ~PackedCounterArray() { + if (isAllocated()) + unmap(reinterpret_cast<void *>(Buffer), BufferSize); + } + + bool isAllocated() const { return !!Buffer; } + + uptr getCount() const { return N; } + + uptr get(uptr I) const { + DCHECK_LT(I, N); + const uptr Index = I >> PackingRatioLog; + const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; + return (Buffer[Index] >> BitOffset) & CounterMask; + } + + void inc(uptr I) const { + DCHECK_LT(get(I), CounterMask); + const uptr Index = I >> PackingRatioLog; + const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; + DCHECK_LT(BitOffset, SCUDO_WORDSIZE); + Buffer[Index] += static_cast<uptr>(1U) << BitOffset; + } + + void incRange(uptr From, uptr To) const { + DCHECK_LE(From, To); + for (uptr I = From; I <= To; I++) + inc(I); + } + + uptr getBufferSize() const { return BufferSize; } + +private: + const uptr N; + uptr CounterSizeBitsLog; + uptr CounterMask; + uptr PackingRatioLog; + uptr BitOffsetMask; + + uptr BufferSize; + uptr *Buffer; +}; + +template <class ReleaseRecorderT> class FreePagesRangeTracker { +public: + explicit FreePagesRangeTracker(ReleaseRecorderT *Recorder) + : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {} + + void processNextPage(bool Freed) { + if (Freed) { + if (!InRange) { + CurrentRangeStatePage = CurrentPage; + InRange = true; + } + } else { + closeOpenedRange(); + } + CurrentPage++; + } + + void finish() { closeOpenedRange(); } + +private: + void closeOpenedRange() { + if (InRange) { + Recorder->releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog), + (CurrentPage << PageSizeLog)); + InRange = false; + } + } + + ReleaseRecorderT *const Recorder; + const uptr PageSizeLog; + bool InRange = false; + uptr CurrentPage = 0; + uptr CurrentRangeStatePage = 0; +}; + +template <class TransferBatchT, class ReleaseRecorderT> +NOINLINE void +releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> *FreeList, uptr Base, + uptr AllocatedPagesCount, uptr BlockSize, + ReleaseRecorderT *Recorder) { + const uptr PageSize = getPageSizeCached(); + + // Figure out the number of chunks per page and whether we can take a fast + // path (the number of chunks per page is the same for all pages). + uptr FullPagesBlockCountMax; + bool SameBlockCountPerPage; + if (BlockSize <= PageSize) { + if (PageSize % BlockSize == 0) { + // Same number of chunks per page, no cross overs. + FullPagesBlockCountMax = PageSize / BlockSize; + SameBlockCountPerPage = true; + } else if (BlockSize % (PageSize % BlockSize) == 0) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks, but all pages contain the same + // number of chunks. + FullPagesBlockCountMax = PageSize / BlockSize + 1; + SameBlockCountPerPage = true; + } else { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks. + FullPagesBlockCountMax = PageSize / BlockSize + 2; + SameBlockCountPerPage = false; + } + } else { + if (BlockSize % PageSize == 0) { + // One chunk covers multiple pages, no cross overs. + FullPagesBlockCountMax = 1; + SameBlockCountPerPage = true; + } else { + // One chunk covers multiple pages, Some chunks are crossing page + // boundaries. Some pages contain one chunk, some contain two. + FullPagesBlockCountMax = 2; + SameBlockCountPerPage = false; + } + } + + PackedCounterArray Counters(AllocatedPagesCount, FullPagesBlockCountMax); + if (!Counters.isAllocated()) + return; + + const uptr PageSizeLog = getLog2(PageSize); + const uptr End = Base + AllocatedPagesCount * PageSize; + + // Iterate over free chunks and count how many free chunks affect each + // allocated page. + if (BlockSize <= PageSize && PageSize % BlockSize == 0) { + // Each chunk affects one page only. + for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { + for (u32 I = 0; I < (*It).getCount(); I++) { + const uptr P = reinterpret_cast<uptr>((*It).get(I)); + if (P >= Base && P < End) + Counters.inc((P - Base) >> PageSizeLog); + } + } + } else { + // In all other cases chunks might affect more than one page. + for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { + for (u32 I = 0; I < (*It).getCount(); I++) { + const uptr P = reinterpret_cast<uptr>((*It).get(I)); + if (P >= Base && P < End) + Counters.incRange((P - Base) >> PageSizeLog, + (P - Base + BlockSize - 1) >> PageSizeLog); + } + } + } + + // Iterate over pages detecting ranges of pages with chunk Counters equal + // to the expected number of chunks for the particular page. + FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder); + if (SameBlockCountPerPage) { + // Fast path, every page has the same number of chunks affecting it. + for (uptr I = 0; I < Counters.getCount(); I++) + RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax); + } else { + // Slow path, go through the pages keeping count how many chunks affect + // each page. + const uptr Pn = BlockSize < PageSize ? PageSize / BlockSize : 1; + const uptr Pnc = Pn * BlockSize; + // The idea is to increment the current page pointer by the first chunk + // size, middle portion size (the portion of the page covered by chunks + // except the first and the last one) and then the last chunk size, adding + // up the number of chunks on the current page and checking on every step + // whether the page boundary was crossed. + uptr PrevPageBoundary = 0; + uptr CurrentBoundary = 0; + for (uptr I = 0; I < Counters.getCount(); I++) { + const uptr PageBoundary = PrevPageBoundary + PageSize; + uptr BlocksPerPage = Pn; + if (CurrentBoundary < PageBoundary) { + if (CurrentBoundary > PrevPageBoundary) + BlocksPerPage++; + CurrentBoundary += Pnc; + if (CurrentBoundary < PageBoundary) { + BlocksPerPage++; + CurrentBoundary += BlockSize; + } + } + PrevPageBoundary = PageBoundary; + + RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage); + } + } + RangeTracker.finish(); +} + +} // namespace scudo + +#endif // SCUDO_RELEASE_H_ diff --git a/lib/scudo/standalone/report.cc b/lib/scudo/standalone/report.cc new file mode 100644 index 000000000..2e453a10a --- /dev/null +++ b/lib/scudo/standalone/report.cc @@ -0,0 +1,192 @@ +//===-- report.cc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "report.h" + +#include "atomic_helpers.h" +#include "string_utils.h" + +#include <stdarg.h> + +namespace scudo { + +class ScopedErrorReport { +public: + ScopedErrorReport() : Message(512) { Message.append("Scudo ERROR: "); } + void append(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + Message.append(Format, Args); + va_end(Args); + } + NORETURN ~ScopedErrorReport() { + outputRaw(Message.data()); + setAbortMessage(Message.data()); + die(); + } + +private: + ScopedString Message; +}; + +INLINE void NORETURN trap() { __builtin_trap(); } + +// This could potentially be called recursively if a CHECK fails in the reports. +void NORETURN reportCheckFailed(const char *File, int Line, + const char *Condition, u64 Value1, u64 Value2) { + static atomic_u32 NumberOfCalls; + if (atomic_fetch_add(&NumberOfCalls, 1, memory_order_relaxed) > 2) { + // TODO(kostyak): maybe sleep here? + trap(); + } + ScopedErrorReport Report; + Report.append("CHECK failed @ %s:%d %s (%llu, %llu)\n", File, Line, Condition, + Value1, Value2); +} + +// Generic string fatal error message. +void NORETURN reportError(const char *Message) { + ScopedErrorReport Report; + Report.append("%s", Message); +} + +void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) { + ScopedErrorReport Report; + Report.append("invalid value for %s option: '%s'\n", FlagType, Value); +} + +// The checksum of a chunk header is invalid. This could be caused by an +// {over,under}write of the header, a pointer that is not an actual chunk. +void NORETURN reportHeaderCorruption(void *Ptr) { + ScopedErrorReport Report; + Report.append("corrupted chunk header at address %p\n", Ptr); +} + +// Two threads have attempted to modify a chunk header at the same time. This is +// symptomatic of a race-condition in the application code, or general lack of +// proper locking. +void NORETURN reportHeaderRace(void *Ptr) { + ScopedErrorReport Report; + Report.append("race on chunk header at address %p\n", Ptr); +} + +// The allocator was compiled with parameters that conflict with field size +// requirements. +void NORETURN reportSanityCheckError(const char *Field) { + ScopedErrorReport Report; + Report.append("maximum possible %s doesn't fit in header\n", Field); +} + +// We enforce a maximum alignment, to keep fields smaller and generally prevent +// integer overflows, or unexpected corner cases. +void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment) { + ScopedErrorReport Report; + Report.append("invalid allocation alignment: %zu exceeds maximum supported " + "alignment of %zu\n", + Alignment, MaxAlignment); +} + +// See above, we also enforce a maximum size. +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize) { + ScopedErrorReport Report; + Report.append("requested allocation size %zu (%zu after adjustments) exceeds " + "maximum supported size of %zu\n", + UserSize, TotalSize, MaxSize); +} + +void NORETURN reportOutOfMemory(uptr RequestedSize) { + ScopedErrorReport Report; + Report.append("out of memory trying to allocate %zu bytes\n", RequestedSize); +} + +static const char *stringifyAction(AllocatorAction Action) { + switch (Action) { + case AllocatorAction::Recycling: + return "recycling"; + case AllocatorAction::Deallocating: + return "deallocating"; + case AllocatorAction::Reallocating: + return "reallocating"; + case AllocatorAction::Sizing: + return "sizing"; + } + return "<invalid action>"; +} + +// The chunk is not in a state congruent with the operation we want to perform. +// This is usually the case with a double-free, a realloc of a freed pointer. +void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr) { + ScopedErrorReport Report; + Report.append("invalid chunk state when %s address %p\n", + stringifyAction(Action), Ptr); +} + +void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr) { + ScopedErrorReport Report; + Report.append("misaligned pointer when %s address %p\n", + stringifyAction(Action), Ptr); +} + +// The deallocation function used is at odds with the one used to allocate the +// chunk (eg: new[]/delete or malloc/delete, and so on). +void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr, + u8 TypeA, u8 TypeB) { + ScopedErrorReport Report; + Report.append("allocation type mismatch when %s address %p (%d vs %d)\n", + stringifyAction(Action), Ptr, TypeA, TypeB); +} + +// The size specified to the delete operator does not match the one that was +// passed to new when allocating the chunk. +void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, + uptr ExpectedSize) { + ScopedErrorReport Report; + Report.append( + "invalid sized delete when deallocating address %p (%zu vs %zu)\n", Ptr, + Size, ExpectedSize); +} + +void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment) { + ScopedErrorReport Report; + Report.append( + "invalid allocation alignment: %zu, alignment must be a power of two\n", + Alignment); +} + +void NORETURN reportCallocOverflow(uptr Count, uptr Size) { + ScopedErrorReport Report; + Report.append("calloc parameters overflow: count * size (%zu * %zu) cannot " + "be represented with type size_t\n", + Count, Size); +} + +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) { + ScopedErrorReport Report; + Report.append( + "invalid alignment requested in posix_memalign: %zu, alignment must be a " + "power of two and a multiple of sizeof(void *) == %zu\n", + Alignment, sizeof(void *)); +} + +void NORETURN reportPvallocOverflow(uptr Size) { + ScopedErrorReport Report; + Report.append("pvalloc parameters overflow: size %zu rounded up to system " + "page size %zu cannot be represented in type size_t\n", + Size, getPageSizeCached()); +} + +void NORETURN reportInvalidAlignedAllocAlignment(uptr Alignment, uptr Size) { + ScopedErrorReport Report; + Report.append("invalid alignment requested in aligned_alloc: %zu, alignment " + "must be a power of two and the requested size %zu must be a " + "multiple of alignment\n", + Alignment, Size); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/report.h b/lib/scudo/standalone/report.h new file mode 100644 index 000000000..14e4e799b --- /dev/null +++ b/lib/scudo/standalone/report.h @@ -0,0 +1,57 @@ +//===-- report.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_REPORT_H_ +#define SCUDO_REPORT_H_ + +#include "internal_defs.h" + +namespace scudo { + +// Reports are *fatal* unless stated otherwise. + +// Generic error. +void NORETURN reportError(const char *Message); + +// Flags related errors. +void NORETURN reportInvalidFlag(const char *FlagType, const char *Value); + +// Chunk header related errors. +void NORETURN reportHeaderCorruption(void *Ptr); +void NORETURN reportHeaderRace(void *Ptr); + +// Sanity checks related error. +void NORETURN reportSanityCheckError(const char *Field); + +// Combined allocator errors. +void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment); +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize); +void NORETURN reportOutOfMemory(uptr RequestedSize); +enum class AllocatorAction : u8 { + Recycling, + Deallocating, + Reallocating, + Sizing, +}; +void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr); +void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr); +void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr, + u8 TypeA, u8 TypeB); +void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, uptr ExpectedSize); + +// C wrappers errors. +void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment); +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment); +void NORETURN reportCallocOverflow(uptr Count, uptr Size); +void NORETURN reportPvallocOverflow(uptr Size); +void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment); + +} // namespace scudo + +#endif // SCUDO_REPORT_H_ diff --git a/lib/scudo/standalone/secondary.cc b/lib/scudo/standalone/secondary.cc new file mode 100644 index 000000000..c0de268be --- /dev/null +++ b/lib/scudo/standalone/secondary.cc @@ -0,0 +1,136 @@ +//===-- secondary.cc --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "secondary.h" + +#include "string_utils.h" + +namespace scudo { + +// As with the Primary, the size passed to this function includes any desired +// alignment, so that the frontend can align the user allocation. The hint +// parameter allows us to unmap spurious memory when dealing with larger +// (greater than a page) alignments on 32-bit platforms. +// Due to the sparsity of address space available on those platforms, requesting +// an allocation from the Secondary with a large alignment would end up wasting +// VA space (even though we are not committing the whole thing), hence the need +// to trim off some of the reserved space. +// For allocations requested with an alignment greater than or equal to a page, +// the committed memory will amount to something close to Size - AlignmentHint +// (pending rounding and headers). +void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) { + DCHECK_GT(Size, AlignmentHint); + const uptr PageSize = getPageSizeCached(); + const uptr MapSize = + roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize) + 2 * PageSize; + MapPlatformData Data = {}; + uptr MapBase = + reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary", + MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + if (!MapBase) + return nullptr; + uptr CommitBase = MapBase + PageSize; + uptr MapEnd = MapBase + MapSize; + + // In the unlikely event of alignments larger than a page, adjust the amount + // of memory we want to commit, and trim the extra memory. + if (AlignmentHint >= PageSize) { + // For alignments greater than or equal to a page, the user pointer (eg: the + // pointer that is returned by the C or C++ allocation APIs) ends up on a + // page boundary , and our headers will live in the preceding page. + CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; + const uptr NewMapBase = CommitBase - PageSize; + DCHECK_GE(NewMapBase, MapBase); + // We only trim the extra memory on 32-bit platforms: 64-bit platforms + // are less constrained memory wise, and that saves us two syscalls. + if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { + unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data); + MapBase = NewMapBase; + } + const uptr NewMapEnd = CommitBase + PageSize + + roundUpTo((Size - AlignmentHint), PageSize) + + PageSize; + DCHECK_LE(NewMapEnd, MapEnd); + if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { + unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); + MapEnd = NewMapEnd; + } + } + + const uptr CommitSize = MapEnd - PageSize - CommitBase; + const uptr Ptr = + reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase), + CommitSize, "scudo:secondary", 0, &Data)); + LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr); + H->MapBase = MapBase; + H->MapSize = MapEnd - MapBase; + H->BlockEnd = CommitBase + CommitSize; + H->Data = Data; + { + SpinMutexLock L(&Mutex); + if (!Tail) { + Tail = H; + } else { + Tail->Next = H; + H->Prev = Tail; + Tail = H; + } + AllocatedBytes += CommitSize; + if (LargestSize < CommitSize) + LargestSize = CommitSize; + NumberOfAllocs++; + Stats.add(StatAllocated, CommitSize); + Stats.add(StatMapped, H->MapSize); + } + if (BlockEnd) + *BlockEnd = CommitBase + CommitSize; + return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize()); +} + +void MapAllocator::deallocate(void *Ptr) { + LargeBlock::Header *H = LargeBlock::getHeader(Ptr); + { + SpinMutexLock L(&Mutex); + LargeBlock::Header *Prev = H->Prev; + LargeBlock::Header *Next = H->Next; + if (Prev) { + CHECK_EQ(Prev->Next, H); + Prev->Next = Next; + } + if (Next) { + CHECK_EQ(Next->Prev, H); + Next->Prev = Prev; + } + if (Tail == H) { + CHECK(!Next); + Tail = Prev; + } else { + CHECK(Next); + } + const uptr CommitSize = H->BlockEnd - reinterpret_cast<uptr>(H); + FreedBytes += CommitSize; + NumberOfFrees++; + Stats.sub(StatAllocated, CommitSize); + Stats.sub(StatMapped, H->MapSize); + } + void *Addr = reinterpret_cast<void *>(H->MapBase); + const uptr Size = H->MapSize; + MapPlatformData Data; + Data = H->Data; + unmap(Addr, Size, UNMAP_ALL, &Data); +} + +void MapAllocator::printStats() const { + Printf("Stats: MapAllocator: allocated %zd times (%zdK), freed %zd times " + "(%zdK), remains %zd (%zdK) max %zdM\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, + NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, + LargestSize >> 20); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/secondary.h b/lib/scudo/standalone/secondary.h new file mode 100644 index 000000000..016928cc6 --- /dev/null +++ b/lib/scudo/standalone/secondary.h @@ -0,0 +1,97 @@ +//===-- secondary.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_SECONDARY_H_ +#define SCUDO_SECONDARY_H_ + +#include "common.h" +#include "mutex.h" +#include "stats.h" + +namespace scudo { + +// This allocator wraps the platform allocation primitives, and as such is on +// the slower side and should preferably be used for larger sized allocations. +// Blocks allocated will be preceded and followed by a guard page, and hold +// their own header that is not checksummed: the guard pages and the Combined +// header should be enough for our purpose. + +namespace LargeBlock { + +struct Header { + LargeBlock::Header *Prev; + LargeBlock::Header *Next; + uptr BlockEnd; + uptr MapBase; + uptr MapSize; + MapPlatformData Data; +}; + +constexpr uptr getHeaderSize() { + return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG); +} + +static Header *getHeader(uptr Ptr) { + return reinterpret_cast<Header *>(Ptr - getHeaderSize()); +} + +static Header *getHeader(const void *Ptr) { + return getHeader(reinterpret_cast<uptr>(Ptr)); +} + +} // namespace LargeBlock + +class MapAllocator { +public: + void initLinkerInitialized(GlobalStats *S) { + Stats.initLinkerInitialized(); + if (S) + S->link(&Stats); + } + void init(GlobalStats *S) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(S); + } + + void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr); + + void deallocate(void *Ptr); + + static uptr getBlockEnd(void *Ptr) { + return LargeBlock::getHeader(Ptr)->BlockEnd; + } + + static uptr getBlockSize(void *Ptr) { + return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr); + } + + void printStats() const; + + void disable() { Mutex.lock(); } + + void enable() { Mutex.unlock(); } + + template <typename F> void iterateOverBlocks(F Callback) const { + for (LargeBlock::Header *H = Tail; H != nullptr; H = H->Prev) + Callback(reinterpret_cast<uptr>(H) + LargeBlock::getHeaderSize()); + } + +private: + StaticSpinMutex Mutex; + LargeBlock::Header *Tail; + uptr AllocatedBytes; + uptr FreedBytes; + uptr LargestSize; + u32 NumberOfAllocs; + u32 NumberOfFrees; + LocalStats Stats; +}; + +} // namespace scudo + +#endif // SCUDO_SECONDARY_H_ diff --git a/lib/scudo/standalone/size_class_map.h b/lib/scudo/standalone/size_class_map.h new file mode 100644 index 000000000..b7df54cf8 --- /dev/null +++ b/lib/scudo/standalone/size_class_map.h @@ -0,0 +1,149 @@ +//===-- size_class_map.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_SIZE_CLASS_MAP_H_ +#define SCUDO_SIZE_CLASS_MAP_H_ + +#include "common.h" +#include "string_utils.h" + +namespace scudo { + +// SizeClassMap maps allocation sizes into size classes and back, in an +// efficient table-free manner. +// +// Class 0 is a special class that doesn't abide by the same rules as other +// classes. The allocator uses it to hold batches. +// +// The other sizes are controlled by the template parameters: +// - MinSizeLog: defines the first class as 2^MinSizeLog bytes. +// - MaxSizeLog: defines the last class as 2^MaxSizeLog bytes. +// - MidSizeLog: classes increase with step 2^MinSizeLog from 2^MinSizeLog to +// 2^MidSizeLog bytes. +// - NumBits: the number of non-zero bits in sizes after 2^MidSizeLog. +// eg. with NumBits==3 all size classes after 2^MidSizeLog look like +// 0b1xx0..0 (where x is either 0 or 1). +// +// This class also gives a hint to a thread-caching allocator about the amount +// of chunks that can be cached per-thread: +// - MaxNumCachedHint is a hint for the max number of chunks cached per class. +// - 2^MaxBytesCachedLog is the max number of bytes cached per class. + +template <u8 NumBits, u8 MinSizeLog, u8 MidSizeLog, u8 MaxSizeLog, + u32 MaxNumCachedHintT, u8 MaxBytesCachedLog> +class SizeClassMap { + static const uptr MinSize = 1UL << MinSizeLog; + static const uptr MidSize = 1UL << MidSizeLog; + static const uptr MidClass = MidSize / MinSize; + static const u8 S = NumBits - 1; + static const uptr M = (1UL << S) - 1; + +public: + static const u32 MaxNumCachedHint = MaxNumCachedHintT; + + static const uptr MaxSize = 1UL << MaxSizeLog; + static const uptr NumClasses = + MidClass + ((MaxSizeLog - MidSizeLog) << S) + 1; + COMPILER_CHECK(NumClasses <= 256); + static const uptr LargestClassId = NumClasses - 1; + static const uptr BatchClassId = 0; + + static uptr getSizeByClassId(uptr ClassId) { + DCHECK_NE(ClassId, BatchClassId); + if (ClassId <= MidClass) + return ClassId << MinSizeLog; + ClassId -= MidClass; + const uptr T = MidSize << (ClassId >> S); + return T + (T >> S) * (ClassId & M); + } + + static uptr getClassIdBySize(uptr Size) { + DCHECK_LE(Size, MaxSize); + if (Size <= MidSize) + return (Size + MinSize - 1) >> MinSizeLog; + const uptr L = getMostSignificantSetBitIndex(Size); + const uptr HBits = (Size >> (L - S)) & M; + const uptr LBits = Size & ((1UL << (L - S)) - 1); + const uptr L1 = L - MidSizeLog; + return MidClass + (L1 << S) + HBits + (LBits > 0); + } + + static u32 getMaxCachedHint(uptr Size) { + DCHECK_LE(Size, MaxSize); + DCHECK_NE(Size, 0); + u32 N; + // Force a 32-bit division if the template parameters allow for it. + if (MaxBytesCachedLog > 31 || MaxSizeLog > 31) + N = static_cast<u32>((1UL << MaxBytesCachedLog) / Size); + else + N = (1U << MaxBytesCachedLog) / static_cast<u32>(Size); + return Max(1U, Min(MaxNumCachedHint, N)); + } + + static void print() { + uptr PrevS = 0; + uptr TotalCached = 0; + for (uptr I = 0; I < NumClasses; I++) { + if (I == BatchClassId) + continue; + const uptr S = getSizeByClassId(I); + if (S >= MidSize / 2 && (S & (S - 1)) == 0) + Printf("\n"); + const uptr D = S - PrevS; + const uptr P = PrevS ? (D * 100 / PrevS) : 0; + const uptr L = S ? getMostSignificantSetBitIndex(S) : 0; + const uptr Cached = getMaxCachedHint(S) * S; + Printf( + "C%02zu => S: %zu diff: +%zu %02zu%% L %zu Cached: %zu %zu; id %zu\n", + I, getSizeByClassId(I), D, P, L, getMaxCachedHint(S), Cached, + getClassIdBySize(S)); + TotalCached += Cached; + PrevS = S; + } + Printf("Total Cached: %zu\n", TotalCached); + } + + static void validate() { + for (uptr C = 0; C < NumClasses; C++) { + if (C == BatchClassId) + continue; + const uptr S = getSizeByClassId(C); + CHECK_NE(S, 0U); + CHECK_EQ(getClassIdBySize(S), C); + if (C < LargestClassId) + CHECK_EQ(getClassIdBySize(S + 1), C + 1); + CHECK_EQ(getClassIdBySize(S - 1), C); + CHECK_GT(getSizeByClassId(C), getSizeByClassId(C - 1)); + } + // Do not perform the loop if the maximum size is too large. + if (MaxSizeLog > 19) + return; + for (uptr S = 1; S <= MaxSize; S++) { + const uptr C = getClassIdBySize(S); + CHECK_LT(C, NumClasses); + CHECK_GE(getSizeByClassId(C), S); + if (C > 0) + CHECK_LT(getSizeByClassId(C - 1), S); + } + } +}; + +typedef SizeClassMap<3, 5, 8, 17, 8, 10> DefaultSizeClassMap; + +// TODO(kostyak): further tune class maps for Android & Fuchsia. +#if SCUDO_WORDSIZE == 64U +typedef SizeClassMap<3, 5, 8, 15, 8, 10> SvelteSizeClassMap; +typedef SizeClassMap<3, 5, 8, 16, 14, 12> AndroidSizeClassMap; +#else +typedef SizeClassMap<3, 4, 7, 15, 8, 10> SvelteSizeClassMap; +typedef SizeClassMap<3, 4, 7, 16, 14, 12> AndroidSizeClassMap; +#endif + +} // namespace scudo + +#endif // SCUDO_SIZE_CLASS_MAP_H_ diff --git a/lib/scudo/standalone/stats.h b/lib/scudo/standalone/stats.h new file mode 100644 index 000000000..7fb9c9ed6 --- /dev/null +++ b/lib/scudo/standalone/stats.h @@ -0,0 +1,105 @@ +//===-- stats.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_STATS_H_ +#define SCUDO_STATS_H_ + +#include "atomic_helpers.h" +#include "mutex.h" + +#include <string.h> + +namespace scudo { + +// Memory allocator statistics +enum StatType { StatAllocated, StatMapped, StatCount }; + +typedef uptr StatCounters[StatCount]; + +// Per-thread stats, live in per-thread cache. We use atomics so that the +// numbers themselves are consistent. But we don't use atomic_{add|sub} or a +// lock, because those are expensive operations , and we only care for the stats +// to be "somewhat" correct: eg. if we call GlobalStats::get while a thread is +// LocalStats::add'ing, this is OK, we will still get a meaningful number. +class LocalStats { +public: + void initLinkerInitialized() {} + void init() { memset(this, 0, sizeof(*this)); } + + void add(StatType I, uptr V) { + V += atomic_load_relaxed(&StatsArray[I]); + atomic_store_relaxed(&StatsArray[I], V); + } + + void sub(StatType I, uptr V) { + V = atomic_load_relaxed(&StatsArray[I]) - V; + atomic_store_relaxed(&StatsArray[I], V); + } + + void set(StatType I, uptr V) { atomic_store_relaxed(&StatsArray[I], V); } + + uptr get(StatType I) const { return atomic_load_relaxed(&StatsArray[I]); } + +private: + friend class GlobalStats; + atomic_uptr StatsArray[StatCount]; + LocalStats *Next; + LocalStats *Prev; +}; + +// Global stats, used for aggregation and querying. +class GlobalStats : public LocalStats { +public: + void initLinkerInitialized() { + Next = this; + Prev = this; + } + void init() { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(); + } + + void link(LocalStats *S) { + SpinMutexLock L(&Mutex); + S->Next = Next; + S->Prev = this; + Next->Prev = S; + Next = S; + } + + void unlink(LocalStats *S) { + SpinMutexLock L(&Mutex); + S->Prev->Next = S->Next; + S->Next->Prev = S->Prev; + for (uptr I = 0; I < StatCount; I++) + add(static_cast<StatType>(I), S->get(static_cast<StatType>(I))); + } + + void get(uptr *S) const { + memset(S, 0, StatCount * sizeof(uptr)); + SpinMutexLock L(&Mutex); + const LocalStats *Stats = this; + for (;;) { + for (uptr I = 0; I < StatCount; I++) + S[I] += Stats->get(static_cast<StatType>(I)); + Stats = Stats->Next; + if (Stats == this) + break; + } + // All stats must be non-negative. + for (uptr I = 0; I < StatCount; I++) + S[I] = static_cast<sptr>(S[I]) >= 0 ? S[I] : 0; + } + +private: + mutable StaticSpinMutex Mutex; +}; + +} // namespace scudo + +#endif // SCUDO_STATS_H_ diff --git a/lib/scudo/standalone/string_utils.cc b/lib/scudo/standalone/string_utils.cc new file mode 100644 index 000000000..f0068afc1 --- /dev/null +++ b/lib/scudo/standalone/string_utils.cc @@ -0,0 +1,236 @@ +//===-- string_utils.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "string_utils.h" +#include "common.h" + +#include <ctype.h> +#include <stdarg.h> +#include <string.h> + +namespace scudo { + +static int appendChar(char **Buffer, const char *BufferEnd, char C) { + if (*Buffer < BufferEnd) { + **Buffer = C; + (*Buffer)++; + } + return 1; +} + +// Appends number in a given Base to buffer. If its length is less than +// |MinNumberLength|, it is padded with leading zeroes or spaces, depending +// on the value of |PadWithZero|. +static int appendNumber(char **Buffer, const char *BufferEnd, u64 AbsoluteValue, + u8 Base, u8 MinNumberLength, bool PadWithZero, + bool Negative, bool Upper) { + constexpr uptr MaxLen = 30; + RAW_CHECK(Base == 10 || Base == 16); + RAW_CHECK(Base == 10 || !Negative); + RAW_CHECK(AbsoluteValue || !Negative); + RAW_CHECK(MinNumberLength < MaxLen); + int Res = 0; + if (Negative && MinNumberLength) + --MinNumberLength; + if (Negative && PadWithZero) + Res += appendChar(Buffer, BufferEnd, '-'); + uptr NumBuffer[MaxLen]; + int Pos = 0; + do { + RAW_CHECK_MSG(static_cast<uptr>(Pos) < MaxLen, + "appendNumber buffer overflow"); + NumBuffer[Pos++] = AbsoluteValue % Base; + AbsoluteValue /= Base; + } while (AbsoluteValue > 0); + if (Pos < MinNumberLength) { + memset(&NumBuffer[Pos], 0, + sizeof(NumBuffer[0]) * static_cast<uptr>(MinNumberLength - Pos)); + Pos = MinNumberLength; + } + RAW_CHECK(Pos > 0); + Pos--; + for (; Pos >= 0 && NumBuffer[Pos] == 0; Pos--) { + char c = (PadWithZero || Pos == 0) ? '0' : ' '; + Res += appendChar(Buffer, BufferEnd, c); + } + if (Negative && !PadWithZero) + Res += appendChar(Buffer, BufferEnd, '-'); + for (; Pos >= 0; Pos--) { + char Digit = static_cast<char>(NumBuffer[Pos]); + Digit = static_cast<char>((Digit < 10) ? '0' + Digit + : (Upper ? 'A' : 'a') + Digit - 10); + Res += appendChar(Buffer, BufferEnd, Digit); + } + return Res; +} + +static int appendUnsigned(char **Buffer, const char *BufferEnd, u64 Num, + u8 Base, u8 MinNumberLength, bool PadWithZero, + bool Upper) { + return appendNumber(Buffer, BufferEnd, Num, Base, MinNumberLength, + PadWithZero, /*Negative=*/false, Upper); +} + +static int appendSignedDecimal(char **Buffer, const char *BufferEnd, s64 Num, + u8 MinNumberLength, bool PadWithZero) { + const bool Negative = (Num < 0); + return appendNumber(Buffer, BufferEnd, + static_cast<u64>(Negative ? -Num : Num), 10, + MinNumberLength, PadWithZero, Negative, + /*Upper=*/false); +} + +// Use the fact that explicitly requesting 0 Width (%0s) results in UB and +// interpret Width == 0 as "no Width requested": +// Width == 0 - no Width requested +// Width < 0 - left-justify S within and pad it to -Width chars, if necessary +// Width > 0 - right-justify S, not implemented yet +static int appendString(char **Buffer, const char *BufferEnd, int Width, + int MaxChars, const char *S) { + if (!S) + S = "<null>"; + int Res = 0; + for (; *S; S++) { + if (MaxChars >= 0 && Res >= MaxChars) + break; + Res += appendChar(Buffer, BufferEnd, *S); + } + // Only the left justified strings are supported. + while (Width < -Res) + Res += appendChar(Buffer, BufferEnd, ' '); + return Res; +} + +static int appendPointer(char **Buffer, const char *BufferEnd, u64 ptr_value) { + int Res = 0; + Res += appendString(Buffer, BufferEnd, 0, -1, "0x"); + Res += appendUnsigned(Buffer, BufferEnd, ptr_value, 16, + SCUDO_POINTER_FORMAT_LENGTH, /*PadWithZero=*/true, + /*Upper=*/false); + return Res; +} + +int formatString(char *Buffer, uptr BufferLength, const char *Format, + va_list Args) { + UNUSED static const char *PrintfFormatsHelp = + "Supported formatString formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; " + "%[-]([0-9]*)?(\\.\\*)?s; %c\n"; + RAW_CHECK(Format); + RAW_CHECK(BufferLength > 0); + const char *BufferEnd = &Buffer[BufferLength - 1]; + const char *Cur = Format; + int Res = 0; + for (; *Cur; Cur++) { + if (*Cur != '%') { + Res += appendChar(&Buffer, BufferEnd, *Cur); + continue; + } + Cur++; + const bool LeftJustified = *Cur == '-'; + if (LeftJustified) + Cur++; + bool HaveWidth = (*Cur >= '0' && *Cur <= '9'); + const bool PadWithZero = (*Cur == '0'); + u8 Width = 0; + if (HaveWidth) { + while (*Cur >= '0' && *Cur <= '9') + Width = static_cast<u8>(Width * 10 + *Cur++ - '0'); + } + const bool HavePrecision = (Cur[0] == '.' && Cur[1] == '*'); + int Precision = -1; + if (HavePrecision) { + Cur += 2; + Precision = va_arg(Args, int); + } + const bool HaveZ = (*Cur == 'z'); + Cur += HaveZ; + const bool HaveLL = !HaveZ && (Cur[0] == 'l' && Cur[1] == 'l'); + Cur += HaveLL * 2; + s64 DVal; + u64 UVal; + const bool HaveLength = HaveZ || HaveLL; + const bool HaveFlags = HaveWidth || HaveLength; + // At the moment only %s supports precision and left-justification. + CHECK(!((Precision >= 0 || LeftJustified) && *Cur != 's')); + switch (*Cur) { + case 'd': { + DVal = HaveLL ? va_arg(Args, s64) + : HaveZ ? va_arg(Args, sptr) : va_arg(Args, int); + Res += appendSignedDecimal(&Buffer, BufferEnd, DVal, Width, PadWithZero); + break; + } + case 'u': + case 'x': + case 'X': { + UVal = HaveLL ? va_arg(Args, u64) + : HaveZ ? va_arg(Args, uptr) : va_arg(Args, unsigned); + const bool Upper = (*Cur == 'X'); + Res += appendUnsigned(&Buffer, BufferEnd, UVal, (*Cur == 'u') ? 10 : 16, + Width, PadWithZero, Upper); + break; + } + case 'p': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += appendPointer(&Buffer, BufferEnd, va_arg(Args, uptr)); + break; + } + case 's': { + RAW_CHECK_MSG(!HaveLength, PrintfFormatsHelp); + // Only left-justified Width is supported. + CHECK(!HaveWidth || LeftJustified); + Res += appendString(&Buffer, BufferEnd, LeftJustified ? -Width : Width, + Precision, va_arg(Args, char *)); + break; + } + case 'c': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += + appendChar(&Buffer, BufferEnd, static_cast<char>(va_arg(Args, int))); + break; + } + case '%': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += appendChar(&Buffer, BufferEnd, '%'); + break; + } + default: { + RAW_CHECK_MSG(false, PrintfFormatsHelp); + } + } + } + RAW_CHECK(Buffer <= BufferEnd); + appendChar(&Buffer, BufferEnd + 1, '\0'); + return Res; +} + +void ScopedString::append(const char *Format, va_list Args) { + CHECK_LT(Length, String.size()); + formatString(String.data() + Length, String.size() - Length, Format, Args); + Length += strlen(String.data() + Length); + CHECK_LT(Length, String.size()); +} + +FORMAT(2, 3) +void ScopedString::append(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + append(Format, Args); + va_end(Args); +} + +FORMAT(1, 2) +void Printf(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + ScopedString Msg(512); + Msg.append(Format, Args); + outputRaw(Msg.data()); + va_end(Args); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/string_utils.h b/lib/scudo/standalone/string_utils.h new file mode 100644 index 000000000..aea7b3ffd --- /dev/null +++ b/lib/scudo/standalone/string_utils.h @@ -0,0 +1,42 @@ +//===-- string_utils.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_STRING_UTILS_H_ +#define SCUDO_STRING_UTILS_H_ + +#include "internal_defs.h" +#include "vector.h" + +#include <stdarg.h> + +namespace scudo { + +class ScopedString { +public: + explicit ScopedString(uptr MaxLength) : String(MaxLength), Length(0) { + String[0] = '\0'; + } + uptr length() { return Length; } + const char *data() { return String.data(); } + void clear() { + String[0] = '\0'; + Length = 0; + } + void append(const char *Format, va_list Args); + void append(const char *Format, ...); + +private: + Vector<char> String; + uptr Length; +}; + +void Printf(const char *Format, ...); + +} // namespace scudo + +#endif // SCUDO_STRING_UTILS_H_ diff --git a/lib/scudo/standalone/tests/CMakeLists.txt b/lib/scudo/standalone/tests/CMakeLists.txt new file mode 100644 index 000000000..182d6a26a --- /dev/null +++ b/lib/scudo/standalone/tests/CMakeLists.txt @@ -0,0 +1,69 @@ +include_directories(..) + +add_custom_target(ScudoUnitTests) +set_target_properties(ScudoUnitTests PROPERTIES + FOLDER "Compiler-RT Tests") + +set(SCUDO_UNITTEST_CFLAGS + ${COMPILER_RT_UNITTEST_CFLAGS} + ${COMPILER_RT_GTEST_CFLAGS} + -I${COMPILER_RT_SOURCE_DIR}/include + -I${COMPILER_RT_SOURCE_DIR}/lib + -I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone + -DGTEST_HAS_RTTI=0) + +set(SCUDO_TEST_ARCH ${SCUDO_STANDALONE_SUPPORTED_ARCH}) + +# gtests requires c++ +set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) +foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES}) + list(APPEND LINK_FLAGS -l${lib}) +endforeach() +list(APPEND LINK_FLAGS -pthread) + +set(TEST_HEADERS) +foreach (header ${SCUDO_HEADERS}) + list(APPEND TEST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header}) +endforeach() + +# add_scudo_unittest(<name> +# SOURCES <sources list> +# HEADERS <extra headers list>) +macro(add_scudo_unittest testname) + cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN}) + if(COMPILER_RT_HAS_SCUDO_STANDALONE) + foreach(arch ${SCUDO_TEST_ARCH}) + set(ScudoUnitTestsObjects) + add_library("RTScudoStandalone.test.${arch}" STATIC + $<TARGET_OBJECTS:RTScudoStandalone.${arch}>) + generate_compiler_rt_tests(ScudoUnitTestsObjects ScudoUnitTests + "${testname}-${arch}-Test" ${arch} + SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE} + COMPILE_DEPS ${TEST_HEADERS} + DEPS gtest scudo_standalone + RUNTIME RTScudoStandalone.test.${arch} + CFLAGS ${SCUDO_UNITTEST_CFLAGS} + LINK_FLAGS ${LINK_FLAGS}) + endforeach() + endif() +endmacro() + +set(SCUDO_UNIT_TEST_SOURCES + atomic_test.cc + bytemap_test.cc + checksum_test.cc + flags_test.cc + list_test.cc + map_test.cc + mutex_test.cc + release_test.cc + report_test.cc + secondary_test.cc + size_class_map_test.cc + stats_test.cc + strings_test.cc + vector_test.cc + scudo_unit_test_main.cc) + +add_scudo_unittest(ScudoUnitTest + SOURCES ${SCUDO_UNIT_TEST_SOURCES}) diff --git a/lib/scudo/standalone/tests/atomic_test.cc b/lib/scudo/standalone/tests/atomic_test.cc new file mode 100644 index 000000000..3095451b9 --- /dev/null +++ b/lib/scudo/standalone/tests/atomic_test.cc @@ -0,0 +1,112 @@ +//===-- atomic_test.cc ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/atomic_helpers.h" +#include "gtest/gtest.h" + +namespace scudo { + +template <typename T> struct ValAndMagic { + typename T::Type Magic0; + T A; + typename T::Type Magic1; + + static ValAndMagic<T> *Sink; +}; + +template <typename T> ValAndMagic<T> *ValAndMagic<T>::Sink; + +template <typename T, memory_order LoadMO, memory_order StoreMO> +void checkStoreLoad() { + typedef typename T::Type Type; + ValAndMagic<T> Val; + // Prevent the compiler from scalarizing the struct. + ValAndMagic<T>::Sink = &Val; + // Ensure that surrounding memory is not overwritten. + Val.Magic0 = Val.Magic1 = (Type)-3; + for (u64 I = 0; I < 100; I++) { + // Generate A value that occupies all bytes of the variable. + u64 V = I; + V |= V << 8; + V |= V << 16; + V |= V << 32; + Val.A.ValDoNotUse = (Type)V; + EXPECT_EQ(atomic_load(&Val.A, LoadMO), (Type)V); + Val.A.ValDoNotUse = (Type)-1; + atomic_store(&Val.A, (Type)V, StoreMO); + EXPECT_EQ(Val.A.ValDoNotUse, (Type)V); + } + EXPECT_EQ(Val.Magic0, (Type)-3); + EXPECT_EQ(Val.Magic1, (Type)-3); +} + +TEST(ScudoAtomicTest, AtomicStoreLoad) { + checkStoreLoad<atomic_u8, memory_order_relaxed, memory_order_relaxed>(); + checkStoreLoad<atomic_u8, memory_order_consume, memory_order_relaxed>(); + checkStoreLoad<atomic_u8, memory_order_acquire, memory_order_relaxed>(); + checkStoreLoad<atomic_u8, memory_order_relaxed, memory_order_release>(); + checkStoreLoad<atomic_u8, memory_order_seq_cst, memory_order_seq_cst>(); + + checkStoreLoad<atomic_u16, memory_order_relaxed, memory_order_relaxed>(); + checkStoreLoad<atomic_u16, memory_order_consume, memory_order_relaxed>(); + checkStoreLoad<atomic_u16, memory_order_acquire, memory_order_relaxed>(); + checkStoreLoad<atomic_u16, memory_order_relaxed, memory_order_release>(); + checkStoreLoad<atomic_u16, memory_order_seq_cst, memory_order_seq_cst>(); + + checkStoreLoad<atomic_u32, memory_order_relaxed, memory_order_relaxed>(); + checkStoreLoad<atomic_u32, memory_order_consume, memory_order_relaxed>(); + checkStoreLoad<atomic_u32, memory_order_acquire, memory_order_relaxed>(); + checkStoreLoad<atomic_u32, memory_order_relaxed, memory_order_release>(); + checkStoreLoad<atomic_u32, memory_order_seq_cst, memory_order_seq_cst>(); + + checkStoreLoad<atomic_u64, memory_order_relaxed, memory_order_relaxed>(); + checkStoreLoad<atomic_u64, memory_order_consume, memory_order_relaxed>(); + checkStoreLoad<atomic_u64, memory_order_acquire, memory_order_relaxed>(); + checkStoreLoad<atomic_u64, memory_order_relaxed, memory_order_release>(); + checkStoreLoad<atomic_u64, memory_order_seq_cst, memory_order_seq_cst>(); + + checkStoreLoad<atomic_uptr, memory_order_relaxed, memory_order_relaxed>(); + checkStoreLoad<atomic_uptr, memory_order_consume, memory_order_relaxed>(); + checkStoreLoad<atomic_uptr, memory_order_acquire, memory_order_relaxed>(); + checkStoreLoad<atomic_uptr, memory_order_relaxed, memory_order_release>(); + checkStoreLoad<atomic_uptr, memory_order_seq_cst, memory_order_seq_cst>(); +} + +template <typename T> void checkAtomicCompareExchange() { + typedef typename T::Type Type; + { + Type OldVal = 42; + Type NewVal = 24; + Type V = OldVal; + EXPECT_TRUE(atomic_compare_exchange_strong( + reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed)); + EXPECT_FALSE(atomic_compare_exchange_strong( + reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed)); + EXPECT_EQ(NewVal, OldVal); + } + { + Type OldVal = 42; + Type NewVal = 24; + Type V = OldVal; + EXPECT_TRUE(atomic_compare_exchange_weak(reinterpret_cast<T *>(&V), &OldVal, + NewVal, memory_order_relaxed)); + EXPECT_FALSE(atomic_compare_exchange_weak( + reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed)); + EXPECT_EQ(NewVal, OldVal); + } +} + +TEST(ScudoAtomicTest, AtomicCompareExchangeTest) { + checkAtomicCompareExchange<atomic_u8>(); + checkAtomicCompareExchange<atomic_u16>(); + checkAtomicCompareExchange<atomic_u32>(); + checkAtomicCompareExchange<atomic_u64>(); + checkAtomicCompareExchange<atomic_uptr>(); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/tests/bytemap_test.cc b/lib/scudo/standalone/tests/bytemap_test.cc new file mode 100644 index 000000000..615b946c5 --- /dev/null +++ b/lib/scudo/standalone/tests/bytemap_test.cc @@ -0,0 +1,73 @@ +//===-- bytemap_test.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "bytemap.h" + +#include "gtest/gtest.h" + +#include <string.h> + +template <typename T> void testMap(T &Map, scudo::uptr Size) { + Map.init(); + for (scudo::uptr I = 0; I < Size; I += 7) + Map.set(I, (I % 100) + 1); + for (scudo::uptr J = 0; J < Size; J++) { + if (J % 7) + EXPECT_EQ(Map[J], 0); + else + EXPECT_EQ(Map[J], (J % 100) + 1); + } +} + +TEST(ScudoByteMapTest, FlatByteMap) { + const scudo::uptr Size = 1U << 10; + scudo::FlatByteMap<Size> Map; + testMap(Map, Size); +} + +TEST(ScudoByteMapTest, TwoLevelByteMap) { + const scudo::uptr Size1 = 1U << 6, Size2 = 1U << 12; + scudo::TwoLevelByteMap<Size1, Size2> Map; + testMap(Map, Size1 * Size2); + Map.reset(); +} + +using TestByteMap = scudo::TwoLevelByteMap<1U << 12, 1U << 13>; + +struct TestByteMapParam { + TestByteMap *Map; + scudo::uptr Shard; + scudo::uptr NumberOfShards; +}; + +void *populateByteMap(void *Param) { + TestByteMapParam *P = reinterpret_cast<TestByteMapParam *>(Param); + for (scudo::uptr I = P->Shard; I < P->Map->size(); I += P->NumberOfShards) { + scudo::u8 V = static_cast<scudo::u8>((I % 100) + 1); + P->Map->set(I, V); + EXPECT_EQ((*P->Map)[I], V); + } + return 0; +} + +TEST(ScudoByteMapTest, ThreadedTwoLevelByteMap) { + TestByteMap Map; + Map.init(); + static const scudo::uptr NumberOfThreads = 16U; + pthread_t T[NumberOfThreads]; + TestByteMapParam P[NumberOfThreads]; + for (scudo::uptr I = 0; I < NumberOfThreads; I++) { + P[I].Map = ⤅ + P[I].Shard = I; + P[I].NumberOfShards = NumberOfThreads; + pthread_create(&T[I], 0, populateByteMap, &P[I]); + } + for (scudo::uptr I = 0; I < NumberOfThreads; I++) + pthread_join(T[I], 0); + Map.reset(); +} diff --git a/lib/scudo/standalone/tests/checksum_test.cc b/lib/scudo/standalone/tests/checksum_test.cc new file mode 100644 index 000000000..2e8dc8a7b --- /dev/null +++ b/lib/scudo/standalone/tests/checksum_test.cc @@ -0,0 +1,58 @@ +//===-- checksum_test.cc ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "checksum.h" + +#include "gtest/gtest.h" + +#include <string.h> + +scudo::u16 computeSoftwareChecksum(scudo::u32 Seed, scudo::uptr *Array, + scudo::uptr ArraySize) { + scudo::u16 Checksum = static_cast<scudo::u16>(Seed & 0xffff); + for (scudo::uptr I = 0; I < ArraySize; I++) + Checksum = scudo::computeBSDChecksum(Checksum, Array[I]); + return Checksum; +} + +scudo::u16 computeHardwareChecksum(scudo::u32 Seed, scudo::uptr *Array, + scudo::uptr ArraySize) { + scudo::u32 Crc = Seed; + for (scudo::uptr I = 0; I < ArraySize; I++) + Crc = scudo::computeHardwareCRC32(Crc, Array[I]); + return static_cast<scudo::u16>((Crc & 0xffff) ^ (Crc >> 16)); +} + +typedef scudo::u16 (*ComputeChecksum)(scudo::u32, scudo::uptr *, scudo::uptr); + +// This verifies that flipping bits in the data being checksummed produces a +// different checksum. We do not use random data to avoid flakyness. +template <ComputeChecksum F> void verifyChecksumFunctionBitFlip() { + scudo::uptr Array[sizeof(scudo::u64) / sizeof(scudo::uptr)]; + const scudo::uptr ArraySize = ARRAY_SIZE(Array); + memset(Array, 0xaa, sizeof(Array)); + const scudo::u32 Seed = 0x41424343U; + const scudo::u16 Reference = F(Seed, Array, ArraySize); + scudo::u8 IdenticalChecksums = 0; + for (scudo::uptr I = 0; I < ArraySize; I++) { + for (scudo::uptr J = 0; J < SCUDO_WORDSIZE; J++) { + Array[I] ^= 1U << J; + if (F(Seed, Array, ArraySize) == Reference) + IdenticalChecksums++; + Array[I] ^= 1U << J; + } + } + // Allow for a couple of identical checksums over the whole set of flips. + EXPECT_LE(IdenticalChecksums, 2); +} + +TEST(ScudoChecksumTest, ChecksumFunctions) { + verifyChecksumFunctionBitFlip<computeSoftwareChecksum>(); + if (&scudo::computeHardwareCRC32 && scudo::hasHardwareCRC32()) + verifyChecksumFunctionBitFlip<computeHardwareChecksum>(); +} diff --git a/lib/scudo/standalone/tests/flags_test.cc b/lib/scudo/standalone/tests/flags_test.cc new file mode 100644 index 000000000..2808a46cd --- /dev/null +++ b/lib/scudo/standalone/tests/flags_test.cc @@ -0,0 +1,119 @@ +//===-- flags_test.cc -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "flags.h" +#include "flags_parser.h" + +#include "gtest/gtest.h" + +#include <string.h> + +static const char FlagName[] = "flag_name"; +static const char FlagDesc[] = "flag description"; + +template <typename T> +static void testFlag(scudo::FlagType Type, T StartValue, const char *Env, + T FinalValue) { + scudo::FlagParser Parser; + T Flag = StartValue; + Parser.registerFlag(FlagName, FlagDesc, Type, &Flag); + Parser.parseString(Env); + EXPECT_EQ(FinalValue, Flag); + // Reporting unrecognized flags is needed to reset them. + scudo::reportUnrecognizedFlags(); +} + +TEST(ScudoFlagsTest, BooleanFlags) { + testFlag(scudo::FlagType::FT_bool, false, "flag_name=1", true); + testFlag(scudo::FlagType::FT_bool, false, "flag_name=yes", true); + testFlag(scudo::FlagType::FT_bool, false, "flag_name='yes'", true); + testFlag(scudo::FlagType::FT_bool, false, "flag_name=true", true); + testFlag(scudo::FlagType::FT_bool, true, "flag_name=0", false); + testFlag(scudo::FlagType::FT_bool, true, "flag_name=\"0\"", false); + testFlag(scudo::FlagType::FT_bool, true, "flag_name=no", false); + testFlag(scudo::FlagType::FT_bool, true, "flag_name=false", false); + testFlag(scudo::FlagType::FT_bool, true, "flag_name='false'", false); +} + +TEST(ScudoFlagsDeathTest, BooleanFlags) { + EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name", true), + "expected '='"); + EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=", true), + "invalid value for bool option: ''"); + EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=2", true), + "invalid value for bool option: '2'"); + EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=-1", true), + "invalid value for bool option: '-1'"); + EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=on", true), + "invalid value for bool option: 'on'"); +} + +TEST(ScudoFlagsTest, IntFlags) { + testFlag(scudo::FlagType::FT_int, -11, nullptr, -11); + testFlag(scudo::FlagType::FT_int, -11, "flag_name=0", 0); + testFlag(scudo::FlagType::FT_int, -11, "flag_name='0'", 0); + testFlag(scudo::FlagType::FT_int, -11, "flag_name=42", 42); + testFlag(scudo::FlagType::FT_int, -11, "flag_name=-42", -42); + testFlag(scudo::FlagType::FT_int, -11, "flag_name=\"-42\"", -42); + + // Unrecognized flags are ignored. + testFlag(scudo::FlagType::FT_int, -11, "--flag_name=42", -11); + testFlag(scudo::FlagType::FT_int, -11, "zzzzzzz=42", -11); +} + +TEST(ScudoFlagsDeathTest, IntFlags) { + EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name", 0), + "expected '='"); + EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name=42U", 0), + "invalid value for int option"); +} + +static void testTwoFlags(const char *Env, bool ExpectedFlag1, + const int ExpectedFlag2, const char *Name1 = "flag1", + const char *Name2 = "flag2") { + scudo::FlagParser Parser; + bool Flag1 = !ExpectedFlag1; + int Flag2; + Parser.registerFlag(Name1, FlagDesc, scudo::FlagType::FT_bool, &Flag1); + Parser.registerFlag(Name2, FlagDesc, scudo::FlagType::FT_int, &Flag2); + Parser.parseString(Env); + EXPECT_EQ(ExpectedFlag1, Flag1); + EXPECT_EQ(Flag2, ExpectedFlag2); + // Reporting unrecognized flags is needed to reset them. + scudo::reportUnrecognizedFlags(); +} + +TEST(ScudoFlagsTest, MultipleFlags) { + testTwoFlags("flag1=1 flag2=42", true, 42); + testTwoFlags("flag2=-1 flag1=0", false, -1); + testTwoFlags("flag1=false:flag2=1337", false, 1337); + testTwoFlags("flag2=42:flag1=yes", true, 42); + testTwoFlags("flag2=42\nflag1=yes", true, 42); + testTwoFlags("flag2=42\r\nflag1=yes", true, 42); + testTwoFlags("flag2=42\tflag1=yes", true, 42); +} + +TEST(ScudoFlagsTest, CommonSuffixFlags) { + testTwoFlags("flag=1 other_flag=42", true, 42, "flag", "other_flag"); + testTwoFlags("other_flag=42 flag=1", true, 42, "flag", "other_flag"); +} + +TEST(ScudoFlagsTest, AllocatorFlags) { + scudo::FlagParser Parser; + scudo::Flags Flags; + scudo::registerFlags(&Parser, &Flags); + Flags.setDefaults(); + Flags.dealloc_type_mismatch = false; + Flags.delete_size_mismatch = false; + Flags.quarantine_max_chunk_size = 1024; + Parser.parseString("dealloc_type_mismatch=true:delete_size_mismatch=true:" + "quarantine_max_chunk_size=2048"); + EXPECT_TRUE(Flags.dealloc_type_mismatch); + EXPECT_TRUE(Flags.delete_size_mismatch); + EXPECT_EQ(2048, Flags.quarantine_max_chunk_size); +} diff --git a/lib/scudo/standalone/tests/list_test.cc b/lib/scudo/standalone/tests/list_test.cc new file mode 100644 index 000000000..e4053d8b0 --- /dev/null +++ b/lib/scudo/standalone/tests/list_test.cc @@ -0,0 +1,185 @@ +//===-- list_test.cc --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/list.h" +#include "gtest/gtest.h" + +struct ListItem { + ListItem *Next; +}; + +typedef scudo::IntrusiveList<ListItem> List; + +static List StaticList; + +static void setList(List *L, ListItem *X = nullptr, ListItem *Y = nullptr, + ListItem *Z = nullptr) { + L->clear(); + if (X) + L->push_back(X); + if (Y) + L->push_back(Y); + if (Z) + L->push_back(Z); +} + +static void checkList(List *L, ListItem *I1, ListItem *I2 = nullptr, + ListItem *I3 = nullptr, ListItem *I4 = nullptr, + ListItem *I5 = nullptr, ListItem *I6 = nullptr) { + if (I1) { + EXPECT_EQ(L->front(), I1); + L->pop_front(); + } + if (I2) { + EXPECT_EQ(L->front(), I2); + L->pop_front(); + } + if (I3) { + EXPECT_EQ(L->front(), I3); + L->pop_front(); + } + if (I4) { + EXPECT_EQ(L->front(), I4); + L->pop_front(); + } + if (I5) { + EXPECT_EQ(L->front(), I5); + L->pop_front(); + } + if (I6) { + EXPECT_EQ(L->front(), I6); + L->pop_front(); + } + EXPECT_TRUE(L->empty()); +} + +TEST(ScudoListTest, IntrusiveList) { + ListItem Items[6]; + EXPECT_EQ(StaticList.size(), 0U); + + List L; + L.clear(); + + ListItem *X = &Items[0]; + ListItem *Y = &Items[1]; + ListItem *Z = &Items[2]; + ListItem *A = &Items[3]; + ListItem *B = &Items[4]; + ListItem *C = &Items[5]; + + EXPECT_EQ(L.size(), 0U); + L.push_back(X); + EXPECT_EQ(L.size(), 1U); + EXPECT_EQ(L.back(), X); + EXPECT_EQ(L.front(), X); + L.pop_front(); + EXPECT_TRUE(L.empty()); + L.checkConsistency(); + + L.push_front(X); + EXPECT_EQ(L.size(), 1U); + EXPECT_EQ(L.back(), X); + EXPECT_EQ(L.front(), X); + L.pop_front(); + EXPECT_TRUE(L.empty()); + L.checkConsistency(); + + L.push_front(X); + L.push_front(Y); + L.push_front(Z); + EXPECT_EQ(L.size(), 3U); + EXPECT_EQ(L.front(), Z); + EXPECT_EQ(L.back(), X); + L.checkConsistency(); + + L.pop_front(); + EXPECT_EQ(L.size(), 2U); + EXPECT_EQ(L.front(), Y); + EXPECT_EQ(L.back(), X); + L.pop_front(); + L.pop_front(); + EXPECT_TRUE(L.empty()); + L.checkConsistency(); + + L.push_back(X); + L.push_back(Y); + L.push_back(Z); + EXPECT_EQ(L.size(), 3U); + EXPECT_EQ(L.front(), X); + EXPECT_EQ(L.back(), Z); + L.checkConsistency(); + + L.pop_front(); + EXPECT_EQ(L.size(), 2U); + EXPECT_EQ(L.front(), Y); + EXPECT_EQ(L.back(), Z); + L.pop_front(); + L.pop_front(); + EXPECT_TRUE(L.empty()); + L.checkConsistency(); + + L.push_back(X); + L.push_back(Y); + L.push_back(Z); + L.extract(X, Y); + EXPECT_EQ(L.size(), 2U); + EXPECT_EQ(L.front(), X); + EXPECT_EQ(L.back(), Z); + L.checkConsistency(); + L.extract(X, Z); + EXPECT_EQ(L.size(), 1U); + EXPECT_EQ(L.front(), X); + EXPECT_EQ(L.back(), X); + L.checkConsistency(); + L.pop_front(); + EXPECT_TRUE(L.empty()); + + List L1, L2; + L1.clear(); + L2.clear(); + + L1.append_front(&L2); + EXPECT_TRUE(L1.empty()); + EXPECT_TRUE(L2.empty()); + + L1.append_back(&L2); + EXPECT_TRUE(L1.empty()); + EXPECT_TRUE(L2.empty()); + + setList(&L1, X); + checkList(&L1, X); + + setList(&L1, X, Y, Z); + setList(&L2, A, B, C); + L1.append_back(&L2); + checkList(&L1, X, Y, Z, A, B, C); + EXPECT_TRUE(L2.empty()); + + setList(&L1, X, Y); + setList(&L2); + L1.append_front(&L2); + checkList(&L1, X, Y); + EXPECT_TRUE(L2.empty()); +} + +TEST(ScudoListTest, IntrusiveListAppendEmpty) { + ListItem I; + List L; + L.clear(); + L.push_back(&I); + List L2; + L2.clear(); + L.append_back(&L2); + EXPECT_EQ(L.back(), &I); + EXPECT_EQ(L.front(), &I); + EXPECT_EQ(L.size(), 1U); + L.append_front(&L2); + EXPECT_EQ(L.back(), &I); + EXPECT_EQ(L.front(), &I); + EXPECT_EQ(L.size(), 1U); +} diff --git a/lib/scudo/standalone/tests/map_test.cc b/lib/scudo/standalone/tests/map_test.cc new file mode 100644 index 000000000..7c726e947 --- /dev/null +++ b/lib/scudo/standalone/tests/map_test.cc @@ -0,0 +1,65 @@ +//===-- map_test.cc ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.h" + +#include "gtest/gtest.h" + +#include <string.h> + +static const char *MappingName = "scudo:test"; + +TEST(ScudoMapTest, MapNoAccessUnmap) { + const scudo::uptr Size = 4 * scudo::getPageSizeCached(); + scudo::MapPlatformData Data = {}; + void *P = scudo::map(nullptr, Size, MappingName, MAP_NOACCESS, &Data); + EXPECT_NE(P, nullptr); + EXPECT_DEATH(memset(P, 0xaa, Size), ""); + scudo::unmap(P, Size, UNMAP_ALL, &Data); +} + +TEST(ScudoMapTest, MapUnmap) { + const scudo::uptr Size = 4 * scudo::getPageSizeCached(); + scudo::MapPlatformData Data = {}; + void *P = scudo::map(nullptr, Size, MappingName, 0, &Data); + EXPECT_NE(P, nullptr); + memset(P, 0xaa, Size); + scudo::unmap(P, Size, 0, &Data); + EXPECT_DEATH(memset(P, 0xbb, Size), ""); +} + +TEST(ScudoMapTest, MapWithGuardUnmap) { + const scudo::uptr PageSize = scudo::getPageSizeCached(); + const scudo::uptr Size = 4 * PageSize; + scudo::MapPlatformData Data = {}; + void *P = scudo::map(nullptr, Size + 2 * PageSize, MappingName, MAP_NOACCESS, + &Data); + EXPECT_NE(P, nullptr); + void *Q = + reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(P) + PageSize); + EXPECT_EQ(scudo::map(Q, Size, MappingName, 0, &Data), Q); + memset(Q, 0xaa, Size); + EXPECT_DEATH(memset(Q, 0xaa, Size + 1), ""); + scudo::unmap(P, Size + 2 * PageSize, UNMAP_ALL, &Data); +} + +TEST(ScudoMapTest, MapGrowUnmap) { + const scudo::uptr PageSize = scudo::getPageSizeCached(); + const scudo::uptr Size = 4 * PageSize; + scudo::MapPlatformData Data = {}; + void *P = scudo::map(nullptr, Size, MappingName, MAP_NOACCESS, &Data); + EXPECT_NE(P, nullptr); + void *Q = + reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(P) + PageSize); + EXPECT_EQ(scudo::map(Q, PageSize, MappingName, 0, &Data), Q); + memset(Q, 0xaa, PageSize); + Q = reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(Q) + PageSize); + EXPECT_EQ(scudo::map(Q, PageSize, MappingName, 0, &Data), Q); + memset(Q, 0xbb, PageSize); + scudo::unmap(P, Size, UNMAP_ALL, &Data); +} diff --git a/lib/scudo/standalone/tests/mutex_test.cc b/lib/scudo/standalone/tests/mutex_test.cc new file mode 100644 index 000000000..ce33db58b --- /dev/null +++ b/lib/scudo/standalone/tests/mutex_test.cc @@ -0,0 +1,121 @@ +//===-- mutex_test.cc -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mutex.h" + +#include "gtest/gtest.h" + +#include <string.h> + +template <typename MutexType> class TestData { +public: + explicit TestData(MutexType *M) : Mutex(M) { + for (scudo::u32 I = 0; I < Size; I++) + Data[I] = 0; + } + + void write() { + Lock L(Mutex); + T V0 = Data[0]; + for (scudo::u32 I = 0; I < Size; I++) { + EXPECT_EQ(Data[I], V0); + Data[I]++; + } + } + + void tryWrite() { + if (!Mutex->tryLock()) + return; + T V0 = Data[0]; + for (scudo::u32 I = 0; I < Size; I++) { + EXPECT_EQ(Data[I], V0); + Data[I]++; + } + Mutex->unlock(); + } + + void backoff() { + volatile T LocalData[Size] = {}; + for (scudo::u32 I = 0; I < Size; I++) { + LocalData[I]++; + EXPECT_EQ(LocalData[I], 1U); + } + } + +private: + typedef scudo::GenericScopedLock<MutexType> Lock; + static const scudo::u32 Size = 64U; + typedef scudo::u64 T; + MutexType *Mutex; + ALIGNED(SCUDO_CACHE_LINE_SIZE) T Data[Size]; +}; + +const scudo::u32 NumberOfThreads = 8; +#if SCUDO_DEBUG +const scudo::u32 NumberOfIterations = 4 * 1024; +#else +const scudo::u32 NumberOfIterations = 16 * 1024; +#endif + +template <typename MutexType> static void *lockThread(void *Param) { + TestData<MutexType> *Data = reinterpret_cast<TestData<MutexType> *>(Param); + for (scudo::u32 I = 0; I < NumberOfIterations; I++) { + Data->write(); + Data->backoff(); + } + return 0; +} + +template <typename MutexType> static void *tryThread(void *Param) { + TestData<MutexType> *Data = reinterpret_cast<TestData<MutexType> *>(Param); + for (scudo::u32 I = 0; I < NumberOfIterations; I++) { + Data->tryWrite(); + Data->backoff(); + } + return 0; +} + +template <typename MutexType> static void checkLocked(MutexType *M) { + scudo::GenericScopedLock<MutexType> L(M); + M->checkLocked(); +} + +TEST(ScudoMutexTest, SpinMutex) { + scudo::SpinMutex M; + M.init(); + TestData<scudo::SpinMutex> Data(&M); + pthread_t Threads[NumberOfThreads]; + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_create(&Threads[I], 0, lockThread<scudo::SpinMutex>, &Data); + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_join(Threads[I], 0); +} + +TEST(ScudoMutexTest, SpinMutexTry) { + scudo::SpinMutex M; + M.init(); + TestData<scudo::SpinMutex> Data(&M); + pthread_t Threads[NumberOfThreads]; + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_create(&Threads[I], 0, tryThread<scudo::SpinMutex>, &Data); + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_join(Threads[I], 0); +} + +TEST(ScudoMutexTest, BlockingMutex) { + scudo::u64 MutexMemory[1024] = {}; + scudo::BlockingMutex *M = + new (MutexMemory) scudo::BlockingMutex(scudo::LINKER_INITIALIZED); + TestData<scudo::BlockingMutex> Data(M); + pthread_t Threads[NumberOfThreads]; + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_create(&Threads[I], 0, lockThread<scudo::BlockingMutex>, &Data); + for (scudo::u32 I = 0; I < NumberOfThreads; I++) + pthread_join(Threads[I], 0); + checkLocked(M); +} diff --git a/lib/scudo/standalone/tests/release_test.cc b/lib/scudo/standalone/tests/release_test.cc new file mode 100644 index 000000000..2279d5d15 --- /dev/null +++ b/lib/scudo/standalone/tests/release_test.cc @@ -0,0 +1,260 @@ +//===-- release_test.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "list.h" +#include "release.h" +#include "size_class_map.h" + +#include "gtest/gtest.h" + +#include <string.h> + +#include <algorithm> +#include <random> + +TEST(ScudoReleaseTest, PackedCounterArray) { + for (scudo::uptr I = 0; I < SCUDO_WORDSIZE; I++) { + // Various valid counter's max values packed into one word. + scudo::PackedCounterArray Counters2N(1, 1UL << I); + EXPECT_EQ(sizeof(scudo::uptr), Counters2N.getBufferSize()); + // Check the "all bit set" values too. + scudo::PackedCounterArray Counters2N1_1(1, ~0UL >> I); + EXPECT_EQ(sizeof(scudo::uptr), Counters2N1_1.getBufferSize()); + // Verify the packing ratio, the counter is Expected to be packed into the + // closest power of 2 bits. + scudo::PackedCounterArray Counters(SCUDO_WORDSIZE, 1UL << I); + EXPECT_EQ(sizeof(scudo::uptr) * scudo::roundUpToPowerOfTwo(I + 1), + Counters.getBufferSize()); + } + + // Go through 1, 2, 4, 8, .. {32,64} bits per counter. + for (scudo::uptr I = 0; (SCUDO_WORDSIZE >> I) != 0; I++) { + // Make sure counters request one memory page for the buffer. + const scudo::uptr NumCounters = + (scudo::getPageSizeCached() / 8) * (SCUDO_WORDSIZE >> I); + scudo::PackedCounterArray Counters(NumCounters, 1UL << ((1UL << I) - 1)); + Counters.inc(0); + for (scudo::uptr C = 1; C < NumCounters - 1; C++) { + EXPECT_EQ(0UL, Counters.get(C)); + Counters.inc(C); + EXPECT_EQ(1UL, Counters.get(C - 1)); + } + EXPECT_EQ(0UL, Counters.get(NumCounters - 1)); + Counters.inc(NumCounters - 1); + if (I > 0) { + Counters.incRange(0, NumCounters - 1); + for (scudo::uptr C = 0; C < NumCounters; C++) + EXPECT_EQ(2UL, Counters.get(C)); + } + } +} + +class StringRangeRecorder { +public: + std::string ReportedPages; + + StringRangeRecorder() + : PageSizeScaledLog(scudo::getLog2(scudo::getPageSizeCached())) {} + + void releasePageRangeToOS(scudo::uptr From, scudo::uptr To) { + From >>= PageSizeScaledLog; + To >>= PageSizeScaledLog; + EXPECT_LT(From, To); + if (!ReportedPages.empty()) + EXPECT_LT(LastPageReported, From); + ReportedPages.append(From - LastPageReported, '.'); + ReportedPages.append(To - From, 'x'); + LastPageReported = To; + } + +private: + const scudo::uptr PageSizeScaledLog; + scudo::uptr LastPageReported = 0; +}; + +TEST(ScudoReleaseTest, FreePagesRangeTracker) { + // 'x' denotes a page to be released, '.' denotes a page to be kept around. + const char *TestCases[] = { + "", + ".", + "x", + "........", + "xxxxxxxxxxx", + "..............xxxxx", + "xxxxxxxxxxxxxxxxxx.....", + "......xxxxxxxx........", + "xxx..........xxxxxxxxxxxxxxx", + "......xxxx....xxxx........", + "xxx..........xxxxxxxx....xxxxxxx", + "x.x.x.x.x.x.x.x.x.x.x.x.", + ".x.x.x.x.x.x.x.x.x.x.x.x", + ".x.x.x.x.x.x.x.x.x.x.x.x.", + "x.x.x.x.x.x.x.x.x.x.x.x.x", + }; + typedef scudo::FreePagesRangeTracker<StringRangeRecorder> RangeTracker; + + for (auto TestCase : TestCases) { + StringRangeRecorder Recorder; + RangeTracker Tracker(&Recorder); + for (scudo::uptr I = 0; TestCase[I] != 0; I++) + Tracker.processNextPage(TestCase[I] == 'x'); + Tracker.finish(); + // Strip trailing '.'-pages before comparing the results as they are not + // going to be reported to range_recorder anyway. + const char *LastX = strrchr(TestCase, 'x'); + std::string Expected(TestCase, + LastX == nullptr ? 0 : (LastX - TestCase + 1)); + EXPECT_STREQ(Expected.c_str(), Recorder.ReportedPages.c_str()); + } +} + +class ReleasedPagesRecorder { +public: + std::set<scudo::uptr> ReportedPages; + + void releasePageRangeToOS(scudo::uptr From, scudo::uptr To) { + const scudo::uptr PageSize = scudo::getPageSizeCached(); + for (scudo::uptr I = From; I < To; I += PageSize) + ReportedPages.insert(I); + } +}; + +// Simplified version of a TransferBatch. +template <class SizeClassMap> struct FreeBatch { + static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint; + void clear() { Count = 0; } + void add(scudo::uptr P) { + DCHECK_LT(Count, MaxCount); + Batch[Count++] = P; + } + scudo::u32 getCount() const { return Count; } + scudo::uptr get(scudo::u32 I) const { + DCHECK_LE(I, Count); + return Batch[I]; + } + FreeBatch *Next; + +private: + scudo::u32 Count; + scudo::uptr Batch[MaxCount]; +}; + +template <class SizeClassMap> void testReleaseFreeMemoryToOS() { + typedef FreeBatch<SizeClassMap> Batch; + const scudo::uptr AllocatedPagesCount = 1024; + const scudo::uptr PageSize = scudo::getPageSizeCached(); + std::mt19937 R; + scudo::u32 RandState = 42; + + for (scudo::uptr I = 1; I <= SizeClassMap::LargestClassId; I++) { + const scudo::uptr BlockSize = SizeClassMap::getSizeByClassId(I); + const scudo::uptr MaxBlocks = AllocatedPagesCount * PageSize / BlockSize; + + // Generate the random free list. + std::vector<scudo::uptr> FreeArray; + bool InFreeRange = false; + scudo::uptr CurrentRangeEnd = 0; + for (scudo::uptr I = 0; I < MaxBlocks; I++) { + if (I == CurrentRangeEnd) { + InFreeRange = (scudo::getRandomU32(&RandState) & 1U) == 1; + CurrentRangeEnd += (scudo::getRandomU32(&RandState) & 0x7f) + 1; + } + if (InFreeRange) + FreeArray.push_back(I * BlockSize); + } + if (FreeArray.empty()) + continue; + // Shuffle the array to ensure that the order is irrelevant. + std::shuffle(FreeArray.begin(), FreeArray.end(), R); + + // Build the FreeList from the FreeArray. + scudo::IntrusiveList<Batch> FreeList; + FreeList.clear(); + Batch *CurrentBatch = nullptr; + for (auto const &Block : FreeArray) { + if (!CurrentBatch) { + CurrentBatch = new Batch; + CurrentBatch->clear(); + FreeList.push_back(CurrentBatch); + } + CurrentBatch->add(Block); + if (CurrentBatch->getCount() == Batch::MaxCount) + CurrentBatch = nullptr; + } + + // Release the memory. + ReleasedPagesRecorder Recorder; + releaseFreeMemoryToOS(&FreeList, 0, AllocatedPagesCount, BlockSize, + &Recorder); + + // Verify that there are no released pages touched by used chunks and all + // ranges of free chunks big enough to contain the entire memory pages had + // these pages released. + scudo::uptr VerifiedReleasedPages = 0; + std::set<scudo::uptr> FreeBlocks(FreeArray.begin(), FreeArray.end()); + + scudo::uptr CurrentBlock = 0; + InFreeRange = false; + scudo::uptr CurrentFreeRangeStart = 0; + for (scudo::uptr I = 0; I <= MaxBlocks; I++) { + const bool IsFreeBlock = + FreeBlocks.find(CurrentBlock) != FreeBlocks.end(); + if (IsFreeBlock) { + if (!InFreeRange) { + InFreeRange = true; + CurrentFreeRangeStart = CurrentBlock; + } + } else { + // Verify that this used chunk does not touch any released page. + const scudo::uptr StartPage = CurrentBlock / PageSize; + const scudo::uptr EndPage = (CurrentBlock + BlockSize - 1) / PageSize; + for (scudo::uptr J = StartPage; J <= EndPage; J++) { + const bool PageReleased = Recorder.ReportedPages.find(J * PageSize) != + Recorder.ReportedPages.end(); + EXPECT_EQ(false, PageReleased); + } + + if (InFreeRange) { + InFreeRange = false; + // Verify that all entire memory pages covered by this range of free + // chunks were released. + scudo::uptr P = scudo::roundUpTo(CurrentFreeRangeStart, PageSize); + while (P + PageSize <= CurrentBlock) { + const bool PageReleased = + Recorder.ReportedPages.find(P) != Recorder.ReportedPages.end(); + EXPECT_EQ(true, PageReleased); + VerifiedReleasedPages++; + P += PageSize; + } + } + } + + CurrentBlock += BlockSize; + } + + EXPECT_EQ(Recorder.ReportedPages.size(), VerifiedReleasedPages); + + while (!FreeList.empty()) { + CurrentBatch = FreeList.front(); + FreeList.pop_front(); + delete CurrentBatch; + } + } +} + +TEST(ScudoReleaseTest, ReleaseFreeMemoryToOSDefault) { + testReleaseFreeMemoryToOS<scudo::DefaultSizeClassMap>(); +} + +TEST(ScudoReleaseTest, ReleaseFreeMemoryToOSAndroid) { + testReleaseFreeMemoryToOS<scudo::AndroidSizeClassMap>(); +} + +TEST(ScudoReleaseTest, ReleaseFreeMemoryToOSSvelte) { + testReleaseFreeMemoryToOS<scudo::SvelteSizeClassMap>(); +} diff --git a/lib/scudo/standalone/tests/report_test.cc b/lib/scudo/standalone/tests/report_test.cc new file mode 100644 index 000000000..ce7eda59c --- /dev/null +++ b/lib/scudo/standalone/tests/report_test.cc @@ -0,0 +1,47 @@ +//===-- report_test.cc ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/report.h" +#include "gtest/gtest.h" + +TEST(ScudoReportTest, Generic) { + void *P = reinterpret_cast<void *>(0x42424242U); + EXPECT_DEATH(scudo::reportError("TEST123"), "Scudo ERROR.*TEST123"); + EXPECT_DEATH(scudo::reportInvalidFlag("ABC", "DEF"), "Scudo ERROR.*ABC.*DEF"); + EXPECT_DEATH(scudo::reportHeaderCorruption(P), "Scudo ERROR.*42424242"); + EXPECT_DEATH(scudo::reportHeaderRace(P), "Scudo ERROR.*42424242"); + EXPECT_DEATH(scudo::reportSanityCheckError("XYZ"), "Scudo ERROR.*XYZ"); + EXPECT_DEATH(scudo::reportAlignmentTooBig(123, 456), "Scudo ERROR.*123.*456"); + EXPECT_DEATH(scudo::reportAllocationSizeTooBig(123, 456, 789), + "Scudo ERROR.*123.*456.*789"); + EXPECT_DEATH(scudo::reportOutOfMemory(4242), "Scudo ERROR.*4242"); + EXPECT_DEATH( + scudo::reportInvalidChunkState(scudo::AllocatorAction::Recycling, P), + "Scudo ERROR.*recycling.*42424242"); + EXPECT_DEATH( + scudo::reportInvalidChunkState(scudo::AllocatorAction::Sizing, P), + "Scudo ERROR.*sizing.*42424242"); + EXPECT_DEATH( + scudo::reportMisalignedPointer(scudo::AllocatorAction::Deallocating, P), + "Scudo ERROR.*deallocating.*42424242"); + EXPECT_DEATH(scudo::reportDeallocTypeMismatch( + scudo::AllocatorAction::Reallocating, P, 0, 1), + "Scudo ERROR.*reallocating.*42424242"); + EXPECT_DEATH(scudo::reportDeleteSizeMismatch(P, 123, 456), + "Scudo ERROR.*42424242.*123.*456"); +} + +TEST(ScudoReportTest, CSpecific) { + EXPECT_DEATH(scudo::reportAlignmentNotPowerOfTwo(123), "Scudo ERROR.*123"); + EXPECT_DEATH(scudo::reportCallocOverflow(123, 456), "Scudo ERROR.*123.*456"); + EXPECT_DEATH(scudo::reportInvalidPosixMemalignAlignment(789), + "Scudo ERROR.*789"); + EXPECT_DEATH(scudo::reportPvallocOverflow(123), "Scudo ERROR.*123"); + EXPECT_DEATH(scudo::reportInvalidAlignedAllocAlignment(123, 456), + "Scudo ERROR.*123.*456"); +} diff --git a/lib/scudo/standalone/tests/scudo_unit_test_main.cc b/lib/scudo/standalone/tests/scudo_unit_test_main.cc new file mode 100644 index 000000000..16398e5da --- /dev/null +++ b/lib/scudo/standalone/tests/scudo_unit_test_main.cc @@ -0,0 +1,14 @@ +//===-- scudo_unit_test_main.cc ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gtest/gtest.h" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/lib/scudo/standalone/tests/secondary_test.cc b/lib/scudo/standalone/tests/secondary_test.cc new file mode 100644 index 000000000..8eed16e0e --- /dev/null +++ b/lib/scudo/standalone/tests/secondary_test.cc @@ -0,0 +1,137 @@ +//===-- secondary_test.cc ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "secondary.h" + +#include "gtest/gtest.h" + +#include <stdio.h> + +#include <condition_variable> +#include <mutex> +#include <thread> + +TEST(ScudoSecondaryTest, SecondaryBasic) { + scudo::GlobalStats S; + S.init(); + scudo::MapAllocator *L = new scudo::MapAllocator; + L->init(&S); + const scudo::uptr Size = 1U << 16; + void *P = L->allocate(Size); + EXPECT_NE(P, nullptr); + memset(P, 'A', Size); + EXPECT_GE(scudo::MapAllocator::getBlockSize(P), Size); + L->deallocate(P); + EXPECT_DEATH(memset(P, 'A', Size), ""); + + const scudo::uptr Align = 1U << 16; + P = L->allocate(Size + Align, Align); + EXPECT_NE(P, nullptr); + void *AlignedP = reinterpret_cast<void *>( + scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); + memset(AlignedP, 'A', Size); + L->deallocate(P); + + std::vector<void *> V; + for (scudo::u8 I = 0; I < 32; I++) + V.push_back(L->allocate(Size)); + std::random_shuffle(V.begin(), V.end()); + while (!V.empty()) { + L->deallocate(V.back()); + V.pop_back(); + } + L->printStats(); +} + +// This exercises a variety of combinations of size and alignment for the +// MapAllocator. The size computation done here mimic the ones done by the +// combined allocator. +TEST(ScudoSecondaryTest, SecondaryCombinations) { + constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16); + constexpr scudo::uptr HeaderSize = scudo::roundUpTo(8, MinAlign); + scudo::MapAllocator *L = new scudo::MapAllocator; + L->init(nullptr); + for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) { + for (scudo::uptr AlignLog = FIRST_32_SECOND_64(3, 4); AlignLog <= 16; + AlignLog++) { + const scudo::uptr Align = 1U << AlignLog; + for (scudo::sptr Delta = -128; Delta <= 128; Delta += 8) { + if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0) + continue; + const scudo::uptr UserSize = + scudo::roundUpTo((1U << SizeLog) + Delta, MinAlign); + const scudo::uptr Size = + HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0); + void *P = L->allocate(Size, Align); + EXPECT_NE(P, nullptr); + void *AlignedP = reinterpret_cast<void *>( + scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); + memset(AlignedP, 0xff, UserSize); + L->deallocate(P); + } + } + } + L->printStats(); +} + +TEST(ScudoSecondaryTest, SecondaryIterate) { + scudo::MapAllocator *L = new scudo::MapAllocator; + L->init(nullptr); + std::vector<void *> V; + const scudo::uptr PageSize = scudo::getPageSizeCached(); + for (scudo::u8 I = 0; I < 32; I++) + V.push_back(L->allocate((std::rand() % 16) * PageSize)); + auto Lambda = [V](scudo::uptr Block) { + EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)), + V.end()); + }; + L->disable(); + L->iterateOverBlocks(Lambda); + L->enable(); + while (!V.empty()) { + L->deallocate(V.back()); + V.pop_back(); + } + L->printStats(); +} + +std::mutex Mutex; +std::condition_variable Cv; +bool Ready = false; + +static void performAllocations(scudo::MapAllocator *L) { + { + std::unique_lock<std::mutex> Lock(Mutex); + while (!Ready) + Cv.wait(Lock); + } + std::vector<void *> V; + const scudo::uptr PageSize = scudo::getPageSizeCached(); + for (scudo::u8 I = 0; I < 32; I++) + V.push_back(L->allocate((std::rand() % 16) * PageSize)); + while (!V.empty()) { + L->deallocate(V.back()); + V.pop_back(); + } +} + +TEST(ScudoSecondaryTest, SecondaryThreadsRace) { + scudo::MapAllocator *L = new scudo::MapAllocator; + L->init(nullptr); + std::thread Threads[10]; + for (scudo::uptr I = 0; I < 10; I++) + Threads[I] = std::thread(performAllocations, L); + { + std::unique_lock<std::mutex> Lock(Mutex); + Ready = true; + Cv.notify_all(); + } + for (auto &T : Threads) + T.join(); + L->printStats(); +} diff --git a/lib/scudo/standalone/tests/size_class_map_test.cc b/lib/scudo/standalone/tests/size_class_map_test.cc new file mode 100644 index 000000000..d857aa4b2 --- /dev/null +++ b/lib/scudo/standalone/tests/size_class_map_test.cc @@ -0,0 +1,38 @@ +//===-- size_class_map_test.cc ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/size_class_map.h" +#include "gtest/gtest.h" + +template <class SizeClassMap> void testSizeClassMap() { + typedef SizeClassMap SCMap; + SCMap::print(); + SCMap::validate(); +} + +TEST(ScudoSizeClassMapTest, DefaultSizeClassMap) { + testSizeClassMap<scudo::DefaultSizeClassMap>(); +} + +TEST(ScudoSizeClassMapTest, SvelteSizeClassMap) { + testSizeClassMap<scudo::SvelteSizeClassMap>(); +} + +TEST(ScudoSizeClassMapTest, AndroidSizeClassMap) { + testSizeClassMap<scudo::AndroidSizeClassMap>(); +} + +TEST(ScudoSizeClassMapTest, OneClassSizeClassMap) { + testSizeClassMap<scudo::SizeClassMap<1, 5, 5, 5, 0, 0>>(); +} + +#if SCUDO_CAN_USE_PRIMARY64 +TEST(ScudoSizeClassMapTest, LargeMaxSizeClassMap) { + testSizeClassMap<scudo::SizeClassMap<3, 4, 8, 63, 128, 16>>(); +} +#endif diff --git a/lib/scudo/standalone/tests/stats_test.cc b/lib/scudo/standalone/tests/stats_test.cc new file mode 100644 index 000000000..9ed105d3d --- /dev/null +++ b/lib/scudo/standalone/tests/stats_test.cc @@ -0,0 +1,45 @@ +//===-- stats_test.cc -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/stats.h" +#include "gtest/gtest.h" + +TEST(ScudoStatsTest, LocalStats) { + scudo::LocalStats LStats; + LStats.init(); + for (scudo::uptr I = 0; I < scudo::StatCount; I++) + EXPECT_EQ(LStats.get(static_cast<scudo::StatType>(I)), 0U); + LStats.add(scudo::StatAllocated, 4096U); + EXPECT_EQ(LStats.get(scudo::StatAllocated), 4096U); + LStats.sub(scudo::StatAllocated, 4096U); + EXPECT_EQ(LStats.get(scudo::StatAllocated), 0U); + LStats.set(scudo::StatAllocated, 4096U); + EXPECT_EQ(LStats.get(scudo::StatAllocated), 4096U); +} + +TEST(ScudoStatsTest, GlobalStats) { + scudo::GlobalStats GStats; + GStats.init(); + scudo::uptr Counters[scudo::StatCount] = {}; + GStats.get(Counters); + for (scudo::uptr I = 0; I < scudo::StatCount; I++) + EXPECT_EQ(Counters[I], 0U); + scudo::LocalStats LStats; + LStats.init(); + GStats.link(&LStats); + for (scudo::uptr I = 0; I < scudo::StatCount; I++) + LStats.add(static_cast<scudo::StatType>(I), 4096U); + GStats.get(Counters); + for (scudo::uptr I = 0; I < scudo::StatCount; I++) + EXPECT_EQ(Counters[I], 4096U); + // Unlinking the local stats move numbers to the global stats. + GStats.unlink(&LStats); + GStats.get(Counters); + for (scudo::uptr I = 0; I < scudo::StatCount; I++) + EXPECT_EQ(Counters[I], 4096U); +} diff --git a/lib/scudo/standalone/tests/strings_test.cc b/lib/scudo/standalone/tests/strings_test.cc new file mode 100644 index 000000000..31e59c403 --- /dev/null +++ b/lib/scudo/standalone/tests/strings_test.cc @@ -0,0 +1,98 @@ +//===-- strings_test.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "scudo/standalone/string_utils.h" +#include "gtest/gtest.h" + +#include <limits.h> + +TEST(ScudoStringsTest, Basic) { + scudo::ScopedString Str(128); + Str.append("a%db%zdc%ue%zuf%xh%zxq%pe%sr", static_cast<int>(-1), + static_cast<scudo::uptr>(-2), static_cast<unsigned>(-4), + static_cast<scudo::uptr>(5), static_cast<unsigned>(10), + static_cast<scudo::uptr>(11), reinterpret_cast<void *>(0x123), + "_string_"); + EXPECT_EQ(Str.length(), strlen(Str.data())); + + std::string expectedString = "a-1b-2c4294967292e5fahbq0x"; + expectedString += std::string(SCUDO_POINTER_FORMAT_LENGTH - 3, '0'); + expectedString += "123e_string_r"; + EXPECT_EQ(Str.length(), strlen(Str.data())); + EXPECT_STREQ(expectedString.c_str(), Str.data()); +} + +TEST(ScudoStringsTest, Precision) { + scudo::ScopedString Str(128); + Str.append("%.*s", 3, "12345"); + EXPECT_EQ(Str.length(), strlen(Str.data())); + EXPECT_STREQ("123", Str.data()); + Str.clear(); + Str.append("%.*s", 6, "12345"); + EXPECT_EQ(Str.length(), strlen(Str.data())); + EXPECT_STREQ("12345", Str.data()); + Str.clear(); + Str.append("%-6s", "12345"); + EXPECT_EQ(Str.length(), strlen(Str.data())); + EXPECT_STREQ("12345 ", Str.data()); +} + +static void fillString(scudo::ScopedString &Str, scudo::uptr Size) { + for (scudo::uptr I = 0; I < Size; I++) + Str.append("A"); +} + +TEST(ScudoStringTest, PotentialOverflows) { + // Use a ScopedString that spans a page, and attempt to write past the end + // of it with variations of append. The expectation is for nothing to crash. + const scudo::uptr PageSize = scudo::getPageSizeCached(); + scudo::ScopedString Str(PageSize); + Str.clear(); + fillString(Str, 2 * PageSize); + Str.clear(); + fillString(Str, PageSize - 64); + Str.append("%-128s", "12345"); + Str.clear(); + fillString(Str, PageSize - 16); + Str.append("%024x", 12345); + Str.clear(); + fillString(Str, PageSize - 16); + Str.append("EEEEEEEEEEEEEEEEEEEEEEEE"); +} + +template <typename T> +static void testAgainstLibc(const char *Format, T Arg1, T Arg2) { + scudo::ScopedString Str(128); + Str.append(Format, Arg1, Arg2); + char Buffer[128]; + snprintf(Buffer, sizeof(Buffer), Format, Arg1, Arg2); + EXPECT_EQ(Str.length(), strlen(Str.data())); + EXPECT_STREQ(Buffer, Str.data()); +} + +TEST(ScudoStringsTest, MinMax) { + testAgainstLibc<int>("%d-%d", INT_MIN, INT_MAX); + testAgainstLibc<unsigned>("%u-%u", 0, UINT_MAX); + testAgainstLibc<unsigned>("%x-%x", 0, UINT_MAX); + testAgainstLibc<long>("%zd-%zd", LONG_MIN, LONG_MAX); + testAgainstLibc<unsigned long>("%zu-%zu", 0, ULONG_MAX); + testAgainstLibc<unsigned long>("%zx-%zx", 0, ULONG_MAX); +} + +TEST(ScudoStringsTest, Padding) { + testAgainstLibc<int>("%3d - %3d", 1, 0); + testAgainstLibc<int>("%3d - %3d", -1, 123); + testAgainstLibc<int>("%3d - %3d", -1, -123); + testAgainstLibc<int>("%3d - %3d", 12, 1234); + testAgainstLibc<int>("%3d - %3d", -12, -1234); + testAgainstLibc<int>("%03d - %03d", 1, 0); + testAgainstLibc<int>("%03d - %03d", -1, 123); + testAgainstLibc<int>("%03d - %03d", -1, -123); + testAgainstLibc<int>("%03d - %03d", 12, 1234); + testAgainstLibc<int>("%03d - %03d", -12, -1234); +} diff --git a/lib/scudo/standalone/tests/vector_test.cc b/lib/scudo/standalone/tests/vector_test.cc new file mode 100644 index 000000000..ebfcc43cc --- /dev/null +++ b/lib/scudo/standalone/tests/vector_test.cc @@ -0,0 +1,43 @@ +//===-- vector_test.cc ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "vector.h" + +#include "gtest/gtest.h" + +TEST(ScudoVectorTest, Basic) { + scudo::Vector<int> V; + EXPECT_EQ(V.size(), 0U); + V.push_back(42); + EXPECT_EQ(V.size(), 1U); + EXPECT_EQ(V[0], 42); + V.push_back(43); + EXPECT_EQ(V.size(), 2U); + EXPECT_EQ(V[0], 42); + EXPECT_EQ(V[1], 43); +} + +TEST(ScudoVectorTest, Stride) { + scudo::Vector<int> V; + for (int i = 0; i < 1000; i++) { + V.push_back(i); + EXPECT_EQ(V.size(), i + 1U); + EXPECT_EQ(V[i], i); + } + for (int i = 0; i < 1000; i++) + EXPECT_EQ(V[i], i); +} + +TEST(ScudoVectorTest, ResizeReduction) { + scudo::Vector<int> V; + V.push_back(0); + V.push_back(0); + EXPECT_EQ(V.size(), 2U); + V.resize(1); + EXPECT_EQ(V.size(), 1U); +} diff --git a/lib/scudo/standalone/vector.h b/lib/scudo/standalone/vector.h new file mode 100644 index 000000000..3cb4005ed --- /dev/null +++ b/lib/scudo/standalone/vector.h @@ -0,0 +1,118 @@ +//===-- vector.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_VECTOR_H_ +#define SCUDO_VECTOR_H_ + +#include "common.h" + +#include <string.h> + +namespace scudo { + +// A low-level vector based on map. May incur a significant memory overhead for +// small vectors. The current implementation supports only POD types. +template <typename T> class VectorNoCtor { +public: + void init(uptr InitialCapacity) { + CapacityBytes = 0; + Size = 0; + Data = nullptr; + reserve(InitialCapacity); + } + void destroy() { + if (Data) + unmap(Data, CapacityBytes); + } + T &operator[](uptr I) { + DCHECK_LT(I, Size); + return Data[I]; + } + const T &operator[](uptr I) const { + DCHECK_LT(I, Size); + return Data[I]; + } + void push_back(const T &Element) { + DCHECK_LE(Size, capacity()); + if (Size == capacity()) { + const uptr NewCapacity = roundUpToPowerOfTwo(Size + 1); + reallocate(NewCapacity); + } + memcpy(&Data[Size++], &Element, sizeof(T)); + } + T &back() { + DCHECK_GT(Size, 0); + return Data[Size - 1]; + } + void pop_back() { + DCHECK_GT(Size, 0); + Size--; + } + uptr size() const { return Size; } + const T *data() const { return Data; } + T *data() { return Data; } + uptr capacity() const { return CapacityBytes / sizeof(T); } + void reserve(uptr NewSize) { + // Never downsize internal buffer. + if (NewSize > capacity()) + reallocate(NewSize); + } + void resize(uptr NewSize) { + if (NewSize > Size) { + reserve(NewSize); + memset(&Data[Size], 0, sizeof(T) * (NewSize - Size)); + } + Size = NewSize; + } + + void clear() { Size = 0; } + bool empty() const { return size() == 0; } + + const T *begin() const { return data(); } + T *begin() { return data(); } + const T *end() const { return data() + size(); } + T *end() { return data() + size(); } + +private: + void reallocate(uptr NewCapacity) { + DCHECK_GT(NewCapacity, 0); + DCHECK_LE(Size, NewCapacity); + const uptr NewCapacityBytes = + roundUpTo(NewCapacity * sizeof(T), getPageSizeCached()); + T *NewData = (T *)map(nullptr, NewCapacityBytes, "scudo:vector"); + if (Data) { + memcpy(NewData, Data, Size * sizeof(T)); + unmap(Data, CapacityBytes); + } + Data = NewData; + CapacityBytes = NewCapacityBytes; + } + + T *Data; + uptr CapacityBytes; + uptr Size; +}; + +template <typename T> class Vector : public VectorNoCtor<T> { +public: + Vector() { VectorNoCtor<T>::init(1); } + explicit Vector(uptr Count) { + VectorNoCtor<T>::init(Count); + this->resize(Count); + } + ~Vector() { VectorNoCtor<T>::destroy(); } + // Disallow copies and moves. + Vector(const Vector &) = delete; + Vector &operator=(const Vector &) = delete; + Vector(Vector &&) = delete; + Vector &operator=(Vector &&) = delete; +}; + +} // namespace scudo + +#endif // SCUDO_VECTOR_H_ diff --git a/lib/stats/stats.cc b/lib/stats/stats.cc index c7a9a38a6..8d4a115d1 100644 --- a/lib/stats/stats.cc +++ b/lib/stats/stats.cc @@ -1,9 +1,8 @@ //===-- stats.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/stats/stats.h b/lib/stats/stats.h index 619470677..4b7514d61 100644 --- a/lib/stats/stats.h +++ b/lib/stats/stats.h @@ -1,9 +1,8 @@ //===-- stats.h -------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/stats/stats_client.cc b/lib/stats/stats_client.cc index 5caf09728..0790f5737 100644 --- a/lib/stats/stats_client.cc +++ b/lib/stats/stats_client.cc @@ -1,9 +1,8 @@ //===-- stats_client.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt index e1da319cc..43dbe8644 100644 --- a/lib/tsan/CMakeLists.txt +++ b/lib/tsan/CMakeLists.txt @@ -61,7 +61,6 @@ set(TSAN_CXX_SOURCES if(APPLE) list(APPEND TSAN_SOURCES rtl/tsan_interceptors_mac.cc - rtl/tsan_libdispatch_mac.cc rtl/tsan_platform_mac.cc rtl/tsan_platform_posix.cc) elseif(UNIX) @@ -71,6 +70,11 @@ elseif(UNIX) rtl/tsan_platform_posix.cc) endif() +if(COMPILER_RT_INTERCEPT_LIBDISPATCH) + list(APPEND TSAN_SOURCES rtl/tsan_libdispatch.cc) + list(APPEND TSAN_RTL_CFLAGS ${COMPILER_RT_LIBDISPATCH_CFLAGS}) +endif() + set(TSAN_HEADERS rtl/tsan_clock.h rtl/tsan_defs.h @@ -236,6 +240,7 @@ endif() # Build libcxx instrumented with TSan. if(COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH AND COMPILER_RT_TEST_COMPILER_ID STREQUAL "Clang" AND NOT ANDROID) set(libcxx_tsan_deps) diff --git a/lib/tsan/benchmarks/func_entry_exit.cc b/lib/tsan/benchmarks/func_entry_exit.cc new file mode 100644 index 000000000..5e0ba1d69 --- /dev/null +++ b/lib/tsan/benchmarks/func_entry_exit.cc @@ -0,0 +1,20 @@ +// Synthetic benchmark for __tsan_func_entry/exit (spends ~75% there). + +void foo(bool x); + +int main() { + volatile int kRepeat1 = 1 << 30; + const int kRepeat = kRepeat1; + for (int i = 0; i < kRepeat; i++) + foo(false); +} + +__attribute__((noinline)) void bar(volatile bool x) { + if (x) + foo(x); +} + +__attribute__((noinline)) void foo(bool x) { + if (__builtin_expect(x, false)) + bar(x); +} diff --git a/lib/tsan/benchmarks/mop.cc b/lib/tsan/benchmarks/mop.cc new file mode 100644 index 000000000..e87fab856 --- /dev/null +++ b/lib/tsan/benchmarks/mop.cc @@ -0,0 +1,80 @@ +// Synthetic benchmark for __tsan_read/write{1,2,4,8}. +// As compared to mini_bench_local/shared.cc this benchmark passes through +// deduplication logic (ContainsSameAccess). +// First argument is access size (1, 2, 4, 8). Second optional arg switches +// from writes to reads. + +#include <pthread.h> +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/syscall.h> +#include <sys/time.h> + +template<typename T, bool write> +void* thread(void *arg) { + const int kSize = 2 << 10; + static volatile long data[kSize]; + static volatile long turn; + const int kRepeat = 1 << 17; + const int id = !!arg; + for (int i = 0; i < kRepeat; i++) { + for (;;) { + int t = __atomic_load_n(&turn, __ATOMIC_ACQUIRE); + if (t == id) + break; + syscall(SYS_futex, &turn, FUTEX_WAIT, t, 0, 0, 0); + } + for (int j = 0; j < kSize; j++) { + if (write) { + ((volatile T*)&data[j])[0] = 1; + ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1] = 1; + } else { + T v0 = ((volatile T*)&data[j])[0]; + T v1 = ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1]; + (void)v0; + (void)v1; + } + } + __atomic_store_n(&turn, 1 - id, __ATOMIC_RELEASE); + syscall(SYS_futex, &turn, FUTEX_WAKE, 0, 0, 0, 0); + } + return 0; +} + +template<typename T, bool write> +void test() { + pthread_t th; + pthread_create(&th, 0, thread<T, write>, (void*)1); + thread<T, write>(0); + pthread_join(th, 0); +} + +template<bool write> +void testw(int size) { + switch (size) { + case 1: return test<char, write>(); + case 2: return test<short, write>(); + case 4: return test<int, write>(); + case 8: return test<long long, write>(); + } +} + +int main(int argc, char** argv) { + int size = 8; + bool write = true; + if (argc > 1) { + size = atoi(argv[1]); + if (size != 1 && size != 2 && size != 4 && size != 8) + size = 8; + } + if (argc > 2) + write = false; + printf("%s%d\n", write ? "write" : "read", size); + if (write) + testw<true>(size); + else + testw<false>(size); + return 0; +} diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh index 65c34d466..a2a7e82b4 100755 --- a/lib/tsan/check_analyze.sh +++ b/lib/tsan/check_analyze.sh @@ -34,16 +34,16 @@ check() { fi } -for f in write1 write2 write4 write8 read2 read4; do +for f in write1 write2 write4 write8; do check $f rsp 1 - check $f push 1 - check $f pop 6 + check $f push 2 + check $f pop 16 done -for f in read1 read8; do +for f in read1 read2 read4 read8; do check $f rsp 1 - check $f push 2 - check $f pop 12 + check $f push 3 + check $f pop 24 done for f in func_entry func_exit; do diff --git a/lib/tsan/dd/CMakeLists.txt b/lib/tsan/dd/CMakeLists.txt index f2b8a6d17..c3f5915dd 100644 --- a/lib/tsan/dd/CMakeLists.txt +++ b/lib/tsan/dd/CMakeLists.txt @@ -10,7 +10,7 @@ set(DD_SOURCES dd_interceptors.cc ) -set(DD_LINKLIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(DD_LINKLIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl DD_LINKLIBS) append_list_if(COMPILER_RT_HAS_LIBRT rt DD_LINKLIBS) diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc index a39218f04..35a72eb36 100644 --- a/lib/tsan/dd/dd_interceptors.cc +++ b/lib/tsan/dd/dd_interceptors.cc @@ -1,9 +1,8 @@ //===-- dd_interceptors.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/tsan/dd/dd_rtl.cc b/lib/tsan/dd/dd_rtl.cc index 99b8ee597..3a9aa215d 100644 --- a/lib/tsan/dd/dd_rtl.cc +++ b/lib/tsan/dd/dd_rtl.cc @@ -1,9 +1,8 @@ //===-- dd_rtl.cc ---------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/tsan/dd/dd_rtl.h b/lib/tsan/dd/dd_rtl.h index 9abf17da4..ffe068430 100644 --- a/lib/tsan/dd/dd_rtl.h +++ b/lib/tsan/dd/dd_rtl.h @@ -1,9 +1,8 @@ //===-- dd_rtl.h ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef DD_RTL_H diff --git a/lib/tsan/go/test.c b/lib/tsan/go/test.c index c484774ca..61be48442 100644 --- a/lib/tsan/go/test.c +++ b/lib/tsan/go/test.c @@ -1,9 +1,8 @@ //===-- test.c ------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/go/tsan_go.cc b/lib/tsan/go/tsan_go.cc index 5f2507b7d..dfd1e1da1 100644 --- a/lib/tsan/go/tsan_go.cc +++ b/lib/tsan/go/tsan_go.cc @@ -1,9 +1,8 @@ //===-- tsan_go.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -214,7 +213,7 @@ void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) { ThreadState *thr = AllocGoroutine(); *pthr = thr; int goid = ThreadCreate(parent, (uptr)pc, 0, true); - ThreadStart(thr, goid, 0, /*workerthread*/ false); + ThreadStart(thr, goid, 0, ThreadType::Regular); } void __tsan_go_end(ThreadState *thr) { diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index ef984a45c..685ca5518 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -1,9 +1,8 @@ //===-- tsan_clock.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h index a891d7bbd..6a1d15a2a 100644 --- a/lib/tsan/rtl/tsan_clock.h +++ b/lib/tsan/rtl/tsan_clock.h @@ -1,9 +1,8 @@ //===-- tsan_clock.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_debugging.cc b/lib/tsan/rtl/tsan_debugging.cc index 067aeef97..8579db12b 100644 --- a/lib/tsan/rtl/tsan_debugging.cc +++ b/lib/tsan/rtl/tsan_debugging.cc @@ -1,9 +1,8 @@ //===-- tsan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h index 3c775debf..293d7decc 100644 --- a/lib/tsan/rtl/tsan_defs.h +++ b/lib/tsan/rtl/tsan_defs.h @@ -1,9 +1,8 @@ //===-- tsan_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index 16dbdf391..64fc50e95 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dispatch_defs.h b/lib/tsan/rtl/tsan_dispatch_defs.h new file mode 100644 index 000000000..6f1d1f75f --- /dev/null +++ b/lib/tsan/rtl/tsan_dispatch_defs.h @@ -0,0 +1,66 @@ +//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_DISPATCH_DEFS_H +#define TSAN_DISPATCH_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +typedef struct dispatch_object_s {} *dispatch_object_t; + +#define DISPATCH_DECL(name) \ + typedef struct name##_s : public dispatch_object_s {} *name##_t + +DISPATCH_DECL(dispatch_queue); +DISPATCH_DECL(dispatch_source); +DISPATCH_DECL(dispatch_group); +DISPATCH_DECL(dispatch_data); +DISPATCH_DECL(dispatch_semaphore); +DISPATCH_DECL(dispatch_io); + +typedef void (*dispatch_function_t)(void *arg); +typedef void (^dispatch_block_t)(void); +typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, + int error); + +typedef long dispatch_once_t; // NOLINT +typedef __sanitizer::u64 dispatch_time_t; +typedef int dispatch_fd_t; // NOLINT +typedef unsigned long dispatch_io_type_t; // NOLINT +typedef unsigned long dispatch_io_close_flags_t; // NOLINT + +extern "C" { +void *dispatch_get_context(dispatch_object_t object); +void dispatch_retain(dispatch_object_t object); +void dispatch_release(dispatch_object_t object); + +extern const dispatch_block_t _dispatch_data_destructor_free; +extern const dispatch_block_t _dispatch_data_destructor_munmap; +} // extern "C" + +#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr +#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free +#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap + +#if __has_attribute(noescape) + #define DISPATCH_NOESCAPE __attribute__((__noescape__)) +#else + #define DISPATCH_NOESCAPE +#endif + +// Data types used in dispatch APIs +typedef unsigned long size_t; // NOLINT +typedef unsigned long uintptr_t; // NOLINT +typedef __sanitizer::s64 off_t; +typedef __sanitizer::u16 mode_t; +typedef long long_t; // NOLINT + +#endif // TSAN_DISPATCH_DEFS_H diff --git a/lib/tsan/rtl/tsan_external.cc b/lib/tsan/rtl/tsan_external.cc index 6c0e9477e..ba8bb71be 100644 --- a/lib/tsan/rtl/tsan_external.cc +++ b/lib/tsan/rtl/tsan_external.cc @@ -1,9 +1,8 @@ //===-- tsan_external.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.cc b/lib/tsan/rtl/tsan_fd.cc index f13a7432e..5b562ae68 100644 --- a/lib/tsan/rtl/tsan_fd.cc +++ b/lib/tsan/rtl/tsan_fd.cc @@ -1,9 +1,8 @@ //===-- tsan_fd.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.h b/lib/tsan/rtl/tsan_fd.h index 64dc84f6e..ce4f2f73b 100644 --- a/lib/tsan/rtl/tsan_fd.h +++ b/lib/tsan/rtl/tsan_fd.h @@ -1,9 +1,8 @@ //===-- tsan_fd.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc index 877fc8b81..17cd919b8 100644 --- a/lib/tsan/rtl/tsan_flags.cc +++ b/lib/tsan/rtl/tsan_flags.cc @@ -1,9 +1,8 @@ //===-- tsan_flags.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.h b/lib/tsan/rtl/tsan_flags.h index 66740def5..dbf9345fa 100644 --- a/lib/tsan/rtl/tsan_flags.h +++ b/lib/tsan/rtl/tsan_flags.h @@ -1,9 +1,8 @@ //===-- tsan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index d3b678fdd..bfb74b696 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -1,9 +1,8 @@ //===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.cc b/lib/tsan/rtl/tsan_ignoreset.cc index cdb90d229..b2f657939 100644 --- a/lib/tsan/rtl/tsan_ignoreset.cc +++ b/lib/tsan/rtl/tsan_ignoreset.cc @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.h b/lib/tsan/rtl/tsan_ignoreset.h index e747d819c..3e318bd67 100644 --- a/lib/tsan/rtl/tsan_ignoreset.h +++ b/lib/tsan/rtl/tsan_ignoreset.h @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index 9e49dfe5d..e8908ac98 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -154,7 +153,7 @@ const int SIG_SETMASK = 2; #endif #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ - (!cur_thread()->is_inited) + (cur_thread_init(), !cur_thread()->is_inited) namespace __tsan { struct SignalDesc { @@ -399,7 +398,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -409,7 +408,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -455,7 +454,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -555,6 +554,7 @@ static void LongJmp(ThreadState *thr, uptr *env) { // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { + cur_thread_init(); SetJmp(cur_thread(), sp, mangled_sp); } @@ -665,7 +665,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(size); void *p = 0; { @@ -682,7 +682,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalCalloc(size, n); void *p = 0; { @@ -694,7 +694,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -706,10 +706,23 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { return p; } +TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { + if (in_symbolizer()) + return InternalReallocArray(p, size, n); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + p = user_reallocarray(thr, pc, p, size, n); + } + invoke_malloc_hook(p, size); + return p; +} + TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -719,7 +732,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -808,14 +821,14 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_valloc(thr, pc, sz); @@ -824,7 +837,7 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { uptr PageSize = GetPageSizeCached(); sz = sz ? RoundUpTo(sz, PageSize) : PageSize; return InternalAlloc(sz, nullptr, PageSize); @@ -839,7 +852,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); if (!p) return errno_ENOMEM; @@ -943,6 +956,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { void *param = p->param; int tid = 0; { + cur_thread_init(); ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; @@ -959,7 +973,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); @@ -1051,6 +1065,16 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + { + SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif + } + REAL(pthread_exit)(retval); +} + #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); @@ -1975,6 +1999,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { + cur_thread_init(); ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -2091,7 +2116,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2204,23 +2229,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO -// There interceptors do not seem to be strictly necessary for tsan. -// But we see cases where the interceptors consume 70% of execution time. -// Memory blocks passed to fgetgrent_r are "written to" by tsan several times. -// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each -// function "writes to" the buffer. Then, the same memory is "written to" -// twice, first as buf and then as pwbufp (both of them refer to the same -// addresses). -#undef SANITIZER_INTERCEPT_GETPWENT -#undef SANITIZER_INTERCEPT_GETPWENT_R -#undef SANITIZER_INTERCEPT_FGETPWENT -#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS -#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ @@ -2620,6 +2634,9 @@ static void unreachable() { } #endif +// Define default implementation since interception of libdispatch is optional. +SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} + void InitializeInterceptors() { #if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. @@ -2637,18 +2654,18 @@ void InitializeInterceptors() { InitializeCommonInterceptors(); InitializeSignalInterceptors(); + InitializeLibdispatchInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. - using __interception::GetRealFunctionAddress; - GetRealFunctionAddress(TSAN_STRING_SETJMP, - (uptr*)&REAL(setjmp_symname), 0, 0); - GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, - (uptr*)&REAL(sigsetjmp_symname), 0, 0); + using __interception::InterceptFunction; + InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, + 0); #if !SANITIZER_NETBSD - GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif #endif @@ -2662,6 +2679,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); + TSAN_INTERCEPT(reallocarray); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(munmap); @@ -2677,6 +2695,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index 763b46b88..88d1edd77 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -21,9 +21,17 @@ class ScopedInterceptor { LibIgnore *libignore(); +#if !SANITIZER_GO +INLINE bool in_symbolizer() { + cur_thread_init(); + return UNLIKELY(cur_thread()->in_symbolizer); +} +#endif + } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc index 579c4d0c0..99c6df9df 100644 --- a/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors_mac.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,8 +20,10 @@ #include "tsan_interface_ann.h" #include "sanitizer_common/sanitizer_addrhashmap.h" +#include <errno.h> #include <libkern/OSAtomic.h> #include <objc/objc-sync.h> +#include <sys/ucontext.h> #if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> @@ -30,6 +31,11 @@ typedef long long_t; // NOLINT +extern "C" { +int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); +int setcontext(const ucontext_t *ucp); +} + namespace __tsan { // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, @@ -354,6 +360,31 @@ TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { return result; } +TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { + { + SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); + } + // Bacause of swapcontext() semantics we have no option but to copy its + // impementation here + if (!oucp || !ucp) { + errno = EINVAL; + return -1; + } + ThreadState *thr = cur_thread(); + const int UCF_SWAPPED = 0x80000000; + oucp->uc_onstack &= ~UCF_SWAPPED; + thr->ignore_interceptors++; + int ret = getcontext(oucp); + if (!(oucp->uc_onstack & UCF_SWAPPED)) { + thr->ignore_interceptors--; + if (!ret) { + oucp->uc_onstack |= UCF_SWAPPED; + ret = setcontext(ucp); + } + } + return ret; +} + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/lib/tsan/rtl/tsan_interface.cc b/lib/tsan/rtl/tsan_interface.cc index ad9b1fe9a..508aadb08 100644 --- a/lib/tsan/rtl/tsan_interface.cc +++ b/lib/tsan/rtl/tsan_interface.cc @@ -1,9 +1,8 @@ //===-- tsan_interface.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef u32 uint32_t; typedef u64 uint64_t; void __tsan_init() { + cur_thread_init(); Initialize(cur_thread()); } @@ -124,6 +124,31 @@ void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { __tsan_unaligned_write8(addr); *addr = v; } + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast<ThreadState *>(fiber), name); +} } // extern "C" void __tsan_acquire(void *addr) { diff --git a/lib/tsan/rtl/tsan_interface.h b/lib/tsan/rtl/tsan_interface.h index 203f6b106..fac57809a 100644 --- a/lib/tsan/rtl/tsan_interface.h +++ b/lib/tsan/rtl/tsan_interface.h @@ -1,9 +1,8 @@ //===-- tsan_interface.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -200,7 +199,7 @@ __extension__ typedef __int128 a128; #endif // Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic typedef enum { mo_relaxed, mo_consume, diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index a7922a42e..e141ddbb7 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_ann.h b/lib/tsan/rtl/tsan_interface_ann.h index 963bcc55a..458d61f53 100644 --- a/lib/tsan/rtl/tsan_interface_ann.h +++ b/lib/tsan/rtl/tsan_interface_ann.h @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index d334394f5..a6b7b0f65 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_atomic.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -475,7 +474,7 @@ static morder convert_morder(morder mo) { #define SCOPED_ATOMIC(func, ...) \ ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ diff --git a/lib/tsan/rtl/tsan_interface_inl.h b/lib/tsan/rtl/tsan_interface_inl.h index fff83ee17..bf4a16586 100644 --- a/lib/tsan/rtl/tsan_interface_inl.h +++ b/lib/tsan/rtl/tsan_interface_inl.h @@ -1,9 +1,8 @@ //===-- tsan_interface_inl.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.cc b/lib/tsan/rtl/tsan_interface_java.cc index 75e960e62..9f227f095 100644 --- a/lib/tsan/rtl/tsan_interface_java.cc +++ b/lib/tsan/rtl/tsan_interface_java.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_java.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.h b/lib/tsan/rtl/tsan_interface_java.h index 0bd49ac3c..5ad16e959 100644 --- a/lib/tsan/rtl/tsan_interface_java.h +++ b/lib/tsan/rtl/tsan_interface_java.h @@ -1,9 +1,8 @@ //===-- tsan_interface_java.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch.cc index df22888b3..48ac6a282 100644 --- a/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/lib/tsan/rtl/tsan_libdispatch.cc @@ -1,38 +1,26 @@ -//===-- tsan_libdispatch_mac.cc -------------------------------------------===// +//===-- tsan_libdispatch.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Mac-specific libdispatch (GCD) support. +// Support for intercepting libdispatch (GCD). //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC - #include "sanitizer_common/sanitizer_common.h" #include "interception/interception.h" #include "tsan_interceptors.h" -#include "tsan_platform.h" #include "tsan_rtl.h" -#include <Block.h> -#include <dispatch/dispatch.h> -#include <pthread.h> - -// DISPATCH_NOESCAPE is not defined prior to XCode 8. -#ifndef DISPATCH_NOESCAPE -#define DISPATCH_NOESCAPE -#endif - -typedef long long_t; // NOLINT +#include "BlocksRuntime/Block.h" +#include "tsan_dispatch_defs.h" namespace __tsan { + typedef u16 uint16_t; typedef struct { dispatch_queue_t queue; @@ -42,7 +30,7 @@ typedef struct { bool submitted_synchronously; bool is_barrier_block; uptr non_queue_sync_object; -} tsan_block_context_t; +} block_context_t; // The offsets of different fields of the dispatch_queue_t structure, exported // by libdispatch.dylib. @@ -86,13 +74,11 @@ static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { return tq; } -static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, - dispatch_queue_t queue, - void *orig_context, - dispatch_function_t orig_work) { - tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc_internal(thr, pc, - sizeof(tsan_block_context_t)); +static block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, void *orig_context, + dispatch_function_t orig_work) { + block_context_t *new_context = + (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -111,7 +97,7 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, bool serial_task = context->is_barrier_block || is_queue_serial static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; Acquire(thr, pc, submit_sync); @@ -126,7 +112,7 @@ static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, } static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; if (context->submitted_synchronously) Release(thr, pc, submit_sync); @@ -142,7 +128,7 @@ static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, static void dispatch_callback_wrap(void *param) { SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); - tsan_block_context_t *context = (tsan_block_context_t *)param; + block_context_t *context = (block_context_t *)param; dispatch_sync_pre_execute(thr, pc, context); @@ -166,13 +152,13 @@ static void invoke_and_release_block(void *param) { Block_release(block); } -#define DISPATCH_INTERCEPT_B(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -185,7 +171,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -194,11 +180,11 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } -#define DISPATCH_INTERCEPT_F(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, context, work); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -211,7 +197,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, context, work, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -220,18 +206,21 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } +#define DISPATCH_INTERCEPT(name, barrier) \ + DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ + DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ + DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ + DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) + // We wrap dispatch_async, dispatch_sync and friends where we allocate a new // context, which is used to synchronize (we release the context before // submitting, and the callback acquires it before executing the original // callback). -DISPATCH_INTERCEPT_B(dispatch_async, false) -DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) -DISPATCH_INTERCEPT_F(dispatch_async_f, false) -DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) -DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) -DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) -DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) -DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) +DISPATCH_INTERCEPT(dispatch, false) +DISPATCH_INTERCEPT(dispatch_barrier, true) + +DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, dispatch_function_t work) TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, dispatch_queue_t queue, dispatch_block_t block) { @@ -239,7 +228,7 @@ TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); Release(thr, pc, (uptr)new_context); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); @@ -324,9 +313,12 @@ TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, return result; } +// Used, but not intercepted. +extern "C" void dispatch_group_enter(dispatch_group_t group); + TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); - // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. + // Acquired in the group notification callback in dispatch_group_notify[_f]. Release(thr, pc, (uptr)group); REAL(dispatch_group_leave)(group); } @@ -336,10 +328,10 @@ TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); dispatch_retain(group); dispatch_group_enter(group); - __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); + __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); WRAP(dispatch_async)(queue, ^(void) { block_copy(); - _Block_release(block_copy); + Block_release(block_copy); WRAP(dispatch_group_leave)(group); dispatch_release(group); }); @@ -358,6 +350,9 @@ TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, }); } +DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) + TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, dispatch_queue_t q, dispatch_block_t block) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); @@ -377,7 +372,7 @@ TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, block(); }); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); new_context->is_barrier_block = true; Release(thr, pc, (uptr)new_context); @@ -395,7 +390,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, if (handler == nullptr) return REAL(dispatch_source_set_event_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0 }; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -424,7 +419,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, if (handler == nullptr) return REAL(dispatch_source_set_cancel_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -455,7 +450,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, if (handler == nullptr) return REAL(dispatch_source_set_registration_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -484,34 +479,54 @@ TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); - void *parent_to_child_sync = nullptr; - uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; - void *child_to_parent_sync = nullptr; - uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; - Release(thr, pc, parent_to_child_sync_uptr); + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { SCOPED_INTERCEPTOR_RAW(dispatch_apply); - Acquire(thr, pc, parent_to_child_sync_uptr); + Acquire(thr, pc, parent_to_child_sync); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); block(iteration); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Release(thr, pc, child_to_parent_sync_uptr); + Release(thr, pc, child_to_parent_sync); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); REAL(dispatch_apply)(iterations, queue, new_block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Acquire(thr, pc, child_to_parent_sync_uptr); + Acquire(thr, pc, child_to_parent_sync); +} + +static void invoke_block_iteration(void *param, size_t iteration) { + auto block = (void (^)(size_t)) param; + block(iteration); } TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, dispatch_queue_t queue, void *context, void (*work)(void *, size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + + // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch + // implements dispatch_apply in terms of dispatch_apply_f. + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); work(context, iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); }; - WRAP(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); } DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) @@ -531,7 +546,7 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); uptr submit_sync = (uptr)new_context; Release(thr, pc, submit_sync); @@ -546,7 +561,7 @@ typedef void (^cleanup_handler_t)(int error); TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -563,7 +578,7 @@ TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -580,7 +595,7 @@ TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -599,7 +614,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, dispatch_data_t data, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -617,7 +632,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, dispatch_block_t barrier) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { nullptr, nullptr, &invoke_block, false, false, false, 0}; new_context.non_queue_sync_object = (uptr)channel; new_context.is_barrier_block = true; @@ -637,7 +652,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -662,7 +677,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -687,7 +702,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -722,6 +737,46 @@ TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { return REAL(dispatch_resume)(o); } -} // namespace __tsan +void InitializeLibdispatchInterceptors() { + INTERCEPT_FUNCTION(dispatch_async); + INTERCEPT_FUNCTION(dispatch_async_f); + INTERCEPT_FUNCTION(dispatch_sync); + INTERCEPT_FUNCTION(dispatch_sync_f); + INTERCEPT_FUNCTION(dispatch_barrier_async); + INTERCEPT_FUNCTION(dispatch_barrier_async_f); + INTERCEPT_FUNCTION(dispatch_barrier_sync); + INTERCEPT_FUNCTION(dispatch_barrier_sync_f); + INTERCEPT_FUNCTION(dispatch_after); + INTERCEPT_FUNCTION(dispatch_after_f); + INTERCEPT_FUNCTION(dispatch_once); + INTERCEPT_FUNCTION(dispatch_once_f); + INTERCEPT_FUNCTION(dispatch_semaphore_signal); + INTERCEPT_FUNCTION(dispatch_semaphore_wait); + INTERCEPT_FUNCTION(dispatch_group_wait); + INTERCEPT_FUNCTION(dispatch_group_leave); + INTERCEPT_FUNCTION(dispatch_group_async); + INTERCEPT_FUNCTION(dispatch_group_async_f); + INTERCEPT_FUNCTION(dispatch_group_notify); + INTERCEPT_FUNCTION(dispatch_group_notify_f); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); + INTERCEPT_FUNCTION(dispatch_apply); + INTERCEPT_FUNCTION(dispatch_apply_f); + INTERCEPT_FUNCTION(dispatch_data_create); + INTERCEPT_FUNCTION(dispatch_read); + INTERCEPT_FUNCTION(dispatch_write); + INTERCEPT_FUNCTION(dispatch_io_read); + INTERCEPT_FUNCTION(dispatch_io_write); + INTERCEPT_FUNCTION(dispatch_io_barrier); + INTERCEPT_FUNCTION(dispatch_io_create); + INTERCEPT_FUNCTION(dispatch_io_create_with_path); + INTERCEPT_FUNCTION(dispatch_io_create_with_io); + INTERCEPT_FUNCTION(dispatch_io_close); + INTERCEPT_FUNCTION(dispatch_resume); +} -#endif // SANITIZER_MAC +} // namespace __tsan diff --git a/lib/tsan/rtl/tsan_malloc_mac.cc b/lib/tsan/rtl/tsan_malloc_mac.cc index 3cc30724d..0b874aecb 100644 --- a/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/lib/tsan/rtl/tsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,19 +28,19 @@ using namespace __tsan; void *p = \ user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + if (in_symbolizer()) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) #define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + if (in_symbolizer()) return InternalRealloc(ptr, size); \ SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) #define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + if (in_symbolizer()) return InternalCalloc(count, size); \ SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ - if (cur_thread()->in_symbolizer) { \ + if (in_symbolizer()) { \ void *p = InternalAlloc(size, nullptr, alignment); \ if (!p) return errno_ENOMEM; \ *memptr = p; \ @@ -50,12 +49,12 @@ using namespace __tsan; SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \ int res = user_posix_memalign(thr, pc, memptr, alignment, size); #define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + if (in_symbolizer()) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); @@ -64,6 +63,8 @@ using namespace __tsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __tsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/tsan/rtl/tsan_md5.cc b/lib/tsan/rtl/tsan_md5.cc index 51279c10d..bfe0c17ba 100644 --- a/lib/tsan/rtl/tsan_md5.cc +++ b/lib/tsan/rtl/tsan_md5.cc @@ -1,9 +1,8 @@ //===-- tsan_md5.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,14 @@ static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) { return ptr; } +#undef F +#undef G +#undef H +#undef I +#undef STEP +#undef SET +#undef GET + void MD5_Init(MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index b160a9736..f4a95d870 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -1,9 +1,8 @@ //===-- tsan_mman.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,6 +201,16 @@ void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { return SetErrnoOnNull(p); } +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportReallocArrayOverflow(size, n, &stack); + } + return user_realloc(thr, pc, p, size * n); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h index 6042c5c5d..467aabdf2 100644 --- a/lib/tsan/rtl/tsan_mman.h +++ b/lib/tsan/rtl/tsan_mman.h @@ -1,9 +1,8 @@ //===-- tsan_mman.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); void *user_alloc(ThreadState *thr, uptr pc, uptr sz); void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n); void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, uptr sz); diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc index 22afefce3..bb7531325 100644 --- a/lib/tsan/rtl/tsan_mutex.cc +++ b/lib/tsan/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h index 22ee2f33f..80fdc6ed5 100644 --- a/lib/tsan/rtl/tsan_mutex.h +++ b/lib/tsan/rtl/tsan_mutex.h @@ -1,9 +1,8 @@ //===-- tsan_mutex.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.cc b/lib/tsan/rtl/tsan_mutexset.cc index 21587770f..02e5f9da6 100644 --- a/lib/tsan/rtl/tsan_mutexset.cc +++ b/lib/tsan/rtl/tsan_mutexset.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.h b/lib/tsan/rtl/tsan_mutexset.h index 605c21a9c..d63881f40 100644 --- a/lib/tsan/rtl/tsan_mutexset.h +++ b/lib/tsan/rtl/tsan_mutexset.h @@ -1,9 +1,8 @@ //===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc index 4f52d3d71..4cbdf703a 100644 --- a/lib/tsan/rtl/tsan_new_delete.cc +++ b/lib/tsan/rtl/tsan_new_delete.cc @@ -1,9 +1,8 @@ //===-- tsan_new_delete.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,7 +29,7 @@ DECLARE_REAL(void, free, void *ptr) // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. #define OPERATOR_NEW_BODY(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size); \ void *p = 0; \ { \ @@ -45,7 +44,7 @@ DECLARE_REAL(void, free, void *ptr) return p; #define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, (uptr)align); \ void *p = 0; \ { \ @@ -115,7 +114,7 @@ void *operator new[](__sanitizer::uptr size, std::align_val_t align, #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 8303c2418..03a36fd9e 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -1,9 +1,8 @@ //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index d2ce60709..d6a05386f 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_linux.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -402,6 +401,10 @@ ThreadState *cur_thread() { return thr; } +void set_cur_thread(ThreadState *thr) { + *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); +} + void cur_thread_finalize() { __sanitizer_sigset_t emptyset; internal_sigfillset(&emptyset); diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index 7e3a47387..9b9c8dbe1 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_mac.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,22 +73,22 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { // shadow memory is set up. static uptr main_thread_identity = 0; ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; +static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state; -ThreadState **cur_thread_location() { - ThreadState **thread_identity = (ThreadState **)pthread_self(); - return ((uptr)thread_identity == main_thread_identity) ? nullptr - : thread_identity; +static ThreadState **cur_thread_location() { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity || main_thread_identity == 0) + return &main_thread_state_loc; + return (ThreadState **)MemToShadow(thread_identity); } ThreadState *cur_thread() { - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr || main_thread_identity == 0) { - return (ThreadState *)&main_thread_state; - } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( - (uptr *)fake_tls, sizeof(ThreadState)); - return thr; + return (ThreadState *)SignalSafeGetOrAllocate( + (uptr *)cur_thread_location(), sizeof(ThreadState)); +} + +void set_cur_thread(ThreadState *thr) { + *cur_thread_location() = thr; } // TODO(kuba.brecka): This is not async-signal-safe. In particular, we call @@ -97,14 +96,13 @@ ThreadState *cur_thread() { // handler will try to access the unmapped ThreadState. void cur_thread_finalize() { ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + if (thr_state_loc == &main_thread_state_loc) { // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to // exit the main thread. Let's keep the main thread's ThreadState. return; } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - internal_munmap(*fake_tls, sizeof(ThreadState)); - *fake_tls = nullptr; + internal_munmap(*thr_state_loc, sizeof(ThreadState)); + *thr_state_loc = nullptr; } #endif @@ -213,7 +211,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ true); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { if (thread == pthread_self()) { @@ -266,11 +264,11 @@ void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // The pointer to the ThreadState object is stored in the shadow memory // of the tls. uptr tls_end = tls_addr + tls_size; - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity) { MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size); } else { - uptr thr_state_start = (uptr)thr_state_loc; + uptr thr_state_start = thread_identity; uptr thr_state_end = thr_state_start + sizeof(uptr); CHECK_GE(thr_state_start, tls_addr); CHECK_LE(thr_state_start, tls_addr + tls_size); diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc index c38dcc7f2..3bd0f1bd4 100644 --- a/lib/tsan/rtl/tsan_platform_posix.cc +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_posix.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_windows.cc b/lib/tsan/rtl/tsan_platform_windows.cc index 08aa588a4..037297559 100644 --- a/lib/tsan/rtl/tsan_platform_windows.cc +++ b/lib/tsan/rtl/tsan_platform_windows.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_windows.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_preinit.cc b/lib/tsan/rtl/tsan_preinit.cc index a96618d7d..052b35309 100644 --- a/lib/tsan/rtl/tsan_preinit.cc +++ b/lib/tsan/rtl/tsan_preinit.cc @@ -1,9 +1,8 @@ //===-- tsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc index 629c3e933..ae669024a 100644 --- a/lib/tsan/rtl/tsan_report.cc +++ b/lib/tsan/rtl/tsan_report.cc @@ -1,9 +1,8 @@ //===-- tsan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -258,7 +257,7 @@ static void PrintThread(const ReportThread *rt) { Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; const char *thread_status = rt->running ? "running" : "finished"; - if (rt->workerthread) { + if (rt->thread_type == ThreadType::Worker) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); Printf("%s", d.Default()); diff --git a/lib/tsan/rtl/tsan_report.h b/lib/tsan/rtl/tsan_report.h index cdc999c6a..b4e4d8989 100644 --- a/lib/tsan/rtl/tsan_report.h +++ b/lib/tsan/rtl/tsan_report.h @@ -1,9 +1,8 @@ //===-- tsan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" @@ -92,7 +92,7 @@ struct ReportThread { int id; tid_t os_id; bool running; - bool workerthread; + ThreadType thread_type; char *name; u32 parent_tid; ReportStack *stack; diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index f038e9682..3d23f5047 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -329,11 +328,8 @@ static void CheckShadowMapping() { #if !SANITIZER_GO static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - uptr top = 0; - uptr bottom = 0; - bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { @@ -397,7 +393,7 @@ void Initialize(ThreadState *thr) { // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -642,6 +638,7 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, // __m128i _mm_move_epi64(__m128i*); // _mm_storel_epi64(u64*, __m128i); u64 store_word = cur.raw(); + bool stored = false; // scan all the shadow values and dispatch to 4 categories: // same, replace, candidate and race (see comments below). @@ -666,16 +663,28 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, int idx = 0; #include "tsan_update_shadow_word_inl.h" idx = 1; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 2; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 3; + if (stored) { #include "tsan_update_shadow_word_inl.h" + } else { +#include "tsan_update_shadow_word_inl.h" + } #endif // we did not find any races and had already stored // the current access info, so we are done - if (LIKELY(store_word == 0)) + if (LIKELY(stored)) return; // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); @@ -815,7 +824,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif - if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -826,7 +835,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h index 3410be294..d58c1dca4 100644 --- a/lib/tsan/rtl/tsan_rtl.h +++ b/lib/tsan/rtl/tsan_rtl.h @@ -1,9 +1,8 @@ //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,19 +55,14 @@ namespace __tsan { #if !SANITIZER_GO struct MapUnmapCallback; #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) -static const uptr kAllocatorRegionSizeLog = 20; -static const uptr kAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - LocalAddressSpaceView, MapUnmapCallback>; + struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -85,10 +79,8 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; Allocator *allocator(); #endif @@ -385,6 +377,9 @@ struct ThreadState { // taken by epoch between synchs. // This way we can save one load from tls. u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. @@ -462,12 +457,22 @@ struct ThreadState { #if !SANITIZER_GO #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); +void set_cur_thread(ThreadState *thr); void cur_thread_finalize(); +INLINE void cur_thread_init() { } #else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { - return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); + return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; +} +INLINE void cur_thread_init() { + ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; +} +INLINE void set_cur_thread(ThreadState *thr) { + reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -765,7 +770,8 @@ void FuncEntry(ThreadState *thr, uptr pc); void FuncExit(ThreadState *thr); int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread); +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type); void ThreadFinish(ThreadState *thr); int ThreadTid(ThreadState *thr, uptr pc, uptr uid); void ThreadJoin(ThreadState *thr, uptr pc, int tid); @@ -868,6 +874,16 @@ uptr ALWAYS_INLINE HeapEnd() { } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S index 3d02bf22f..7c3ce13e4 100644 --- a/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -335,9 +335,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif -#if defined(__linux__) -/* We do not need executable stack. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index 34ef51c2a..b5c8cb7bf 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -389,10 +389,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif // !defined(__APPLE__) && !defined(__NetBSD__) -#if defined(__FreeBSD__) || defined(__linux__) -/* We do not need executable stack. */ -/* This note is not needed on NetBSD. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index c61d02b7a..941e70f98 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_mutex.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_proc.cc b/lib/tsan/rtl/tsan_rtl_proc.cc index efccdb590..94bbed25b 100644 --- a/lib/tsan/rtl/tsan_rtl_proc.cc +++ b/lib/tsan/rtl/tsan_rtl_proc.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_proc.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index febb6cef2..220a425a2 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_report.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,7 +201,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; - rt->workerthread = tctx->workerthread; + rt->thread_type = tctx->thread_type; rt->stack = 0; rt->stack = SymbolizeStackId(tctx->creation_stack_id); if (rt->stack) @@ -730,10 +729,12 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { ALWAYS_INLINE void PrintCurrentStackSlow(uptr pc) { #if !SANITIZER_GO + uptr bp = GET_CURRENT_FRAME(); BufferedStackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace))) BufferedStackTrace(); - ptrace->Unwind(kStackTraceMax, pc, 0, 0, 0, 0, false); + ptrace->Unwind(pc, bp, nullptr, false); + for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace_buffer[i]; ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1]; diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 766a0f5a5..fd95cfed4 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_thread.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,13 +239,15 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { return tid; } -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type) { uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -258,7 +259,7 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, workerthread, &args); + tr->StartThread(tid, os_id, thread_type, &args); tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); @@ -404,4 +405,40 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } } +#if !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast<ThreadState *>(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan diff --git a/lib/tsan/rtl/tsan_stack_trace.cc b/lib/tsan/rtl/tsan_stack_trace.cc index a0dee19e2..dbaca23c6 100644 --- a/lib/tsan/rtl/tsan_stack_trace.cc +++ b/lib/tsan/rtl/tsan_stack_trace.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,3 +48,16 @@ void VarSizeStackTrace::ReverseOrder() { } } // namespace __tsan + +#if !SANITIZER_GO +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + uptr top = 0; + uptr bottom = 0; + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} +#endif // SANITIZER_GO diff --git a/lib/tsan/rtl/tsan_stack_trace.h b/lib/tsan/rtl/tsan_stack_trace.h index f69b57464..3eb8ce156 100644 --- a/lib/tsan/rtl/tsan_stack_trace.h +++ b/lib/tsan/rtl/tsan_stack_trace.h @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 18c83d5c6..d23ff47d9 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -1,9 +1,8 @@ //===-- tsan_stat.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 42d6a2b63..94e18bc66 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -1,9 +1,8 @@ //===-- tsan_stat.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.cc b/lib/tsan/rtl/tsan_suppressions.cc index 6df074118..b3eea9ab5 100644 --- a/lib/tsan/rtl/tsan_suppressions.cc +++ b/lib/tsan/rtl/tsan_suppressions.cc @@ -1,9 +1,8 @@ //===-- tsan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.h b/lib/tsan/rtl/tsan_suppressions.h index 526952d5b..f430aeb6c 100644 --- a/lib/tsan/rtl/tsan_suppressions.h +++ b/lib/tsan/rtl/tsan_suppressions.h @@ -1,9 +1,8 @@ //===-- tsan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.cc b/lib/tsan/rtl/tsan_symbolize.cc index 27f0e01c7..cb60763f4 100644 --- a/lib/tsan/rtl/tsan_symbolize.cc +++ b/lib/tsan/rtl/tsan_symbolize.cc @@ -1,9 +1,8 @@ //===-- tsan_symbolize.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.h b/lib/tsan/rtl/tsan_symbolize.h index 5a9710a3c..7adaa04dc 100644 --- a/lib/tsan/rtl/tsan_symbolize.h +++ b/lib/tsan/rtl/tsan_symbolize.h @@ -1,9 +1,8 @@ //===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc index ba3953375..c613b116e 100644 --- a/lib/tsan/rtl/tsan_sync.cc +++ b/lib/tsan/rtl/tsan_sync.cc @@ -1,9 +1,8 @@ //===-- tsan_sync.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h index 9039970bc..47f2739d8 100644 --- a/lib/tsan/rtl/tsan_sync.h +++ b/lib/tsan/rtl/tsan_sync.h @@ -1,9 +1,8 @@ //===-- tsan_sync.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_trace.h b/lib/tsan/rtl/tsan_trace.h index 9aef375cb..fbd0f72db 100644 --- a/lib/tsan/rtl/tsan_trace.h +++ b/lib/tsan/rtl/tsan_trace.h @@ -1,9 +1,8 @@ //===-- tsan_trace.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_update_shadow_word_inl.h b/lib/tsan/rtl/tsan_update_shadow_word_inl.h index 6e3ac2fa1..056c3aa20 100644 --- a/lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ b/lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -1,9 +1,8 @@ //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,31 +17,35 @@ do { const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); - if (old.IsZero()) { + if (LIKELY(old.IsZero())) { StatInc(thr, StatShadowZero); - if (store_word) + if (!stored) { StoreIfNotYetStored(sp, &store_word); - // The above StoreIfNotYetStored could be done unconditionally - // and it even shows 4% gain on synthetic benchmarks (r4307). + stored = true; + } break; } // is the memory access equal to the previous? - if (Shadow::Addr0AndSizeAreEqual(cur, old)) { + if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { StatInc(thr, StatShadowSameSize); // same thread? - if (Shadow::TidsAreEqual(old, cur)) { + if (LIKELY(Shadow::TidsAreEqual(old, cur))) { StatInc(thr, StatShadowSameThread); - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } StatInc(thr, StatShadowAnotherThread); if (HappensBefore(old, thr)) { - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +59,7 @@ do { StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt index 352319f10..7b1ba21c5 100644 --- a/lib/tsan/tests/CMakeLists.txt +++ b/lib/tsan/tests/CMakeLists.txt @@ -14,6 +14,12 @@ set(TSAN_UNITTEST_CFLAGS -DGTEST_HAS_RTTI=0) set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH}) + +set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) +foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES}) + list(APPEND LINK_FLAGS -l${lib}) +endforeach() + if(APPLE) # Create a static library for test dependencies. @@ -33,12 +39,13 @@ if(APPLE) darwin_filter_host_archs(TSAN_SUPPORTED_ARCH TSAN_TEST_ARCH) list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS}) - set(LINK_FLAGS "-lc++") list(APPEND LINK_FLAGS ${DARWIN_osx_LINK_FLAGS}) add_weak_symbols("ubsan" LINK_FLAGS) add_weak_symbols("sanitizer_common" LINK_FLAGS) else() - set(LINK_FLAGS "-fsanitize=thread;-lstdc++;-lm") + list(APPEND LINK_FLAGS -fsanitize=thread) + list(APPEND LINK_FLAGS -lm) + list(APPEND LINK_FLAGS ${COMPILER_RT_TEST_LIBDISPATCH_CFLAGS}) endif() set(TSAN_RTL_HEADERS) diff --git a/lib/tsan/tests/rtl/tsan_bench.cc b/lib/tsan/tests/rtl/tsan_bench.cc index a3cf22f2c..013510114 100644 --- a/lib/tsan/tests/rtl/tsan_bench.cc +++ b/lib/tsan/tests/rtl/tsan_bench.cc @@ -1,9 +1,8 @@ //===-- tsan_bench.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_mop.cc b/lib/tsan/tests/rtl/tsan_mop.cc index f21742825..a5b0bdda3 100644 --- a/lib/tsan/tests/rtl/tsan_mop.cc +++ b/lib/tsan/tests/rtl/tsan_mop.cc @@ -1,9 +1,8 @@ //===-- tsan_mop.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_mutex.cc b/lib/tsan/tests/rtl/tsan_mutex.cc index 4d9c77961..af12e20b1 100644 --- a/lib/tsan/tests/rtl/tsan_mutex.cc +++ b/lib/tsan/tests/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_posix.cc b/lib/tsan/tests/rtl/tsan_posix.cc index e66dab609..d1940452c 100644 --- a/lib/tsan/tests/rtl/tsan_posix.cc +++ b/lib/tsan/tests/rtl/tsan_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_posix.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_posix_util.h b/lib/tsan/tests/rtl/tsan_posix_util.h index 340693ebb..e80039bfb 100644 --- a/lib/tsan/tests/rtl/tsan_posix_util.h +++ b/lib/tsan/tests/rtl/tsan_posix_util.h @@ -1,9 +1,8 @@ //===-- tsan_posix_util.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_string.cc b/lib/tsan/tests/rtl/tsan_string.cc index 75adc6c85..b236d4631 100644 --- a/lib/tsan/tests/rtl/tsan_string.cc +++ b/lib/tsan/tests/rtl/tsan_string.cc @@ -1,9 +1,8 @@ //===-- tsan_string.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test.cc b/lib/tsan/tests/rtl/tsan_test.cc index 842b41750..51a3b2731 100644 --- a/lib/tsan/tests/rtl/tsan_test.cc +++ b/lib/tsan/tests/rtl/tsan_test.cc @@ -1,9 +1,8 @@ //===-- tsan_test.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test_util.h b/lib/tsan/tests/rtl/tsan_test_util.h index 31b1b188f..df535150a 100644 --- a/lib/tsan/tests/rtl/tsan_test_util.h +++ b/lib/tsan/tests/rtl/tsan_test_util.h @@ -1,9 +1,8 @@ //===-- tsan_test_util.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_test_util_posix.cc b/lib/tsan/tests/rtl/tsan_test_util_posix.cc index d00e26dd5..767c8294d 100644 --- a/lib/tsan/tests/rtl/tsan_test_util_posix.cc +++ b/lib/tsan/tests/rtl/tsan_test_util_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_test_util_posix.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/rtl/tsan_thread.cc b/lib/tsan/tests/rtl/tsan_thread.cc index 5646415a7..9e2da912e 100644 --- a/lib/tsan/tests/rtl/tsan_thread.cc +++ b/lib/tsan/tests/rtl/tsan_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_thread.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc index f6230e1be..43d0a012c 100644 --- a/lib/tsan/tests/unit/tsan_clock_test.cc +++ b/lib/tsan/tests/unit/tsan_clock_test.cc @@ -1,9 +1,8 @@ //===-- tsan_clock_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_dense_alloc_test.cc b/lib/tsan/tests/unit/tsan_dense_alloc_test.cc index e848e48c6..8c544b0d7 100644 --- a/lib/tsan/tests/unit/tsan_dense_alloc_test.cc +++ b/lib/tsan/tests/unit/tsan_dense_alloc_test.cc @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc_test.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_flags_test.cc b/lib/tsan/tests/unit/tsan_flags_test.cc index aa8a024f9..ebdb28f6d 100644 --- a/lib/tsan/tests/unit/tsan_flags_test.cc +++ b/lib/tsan/tests/unit/tsan_flags_test.cc @@ -1,9 +1,8 @@ //===-- tsan_flags_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mman_test.cc b/lib/tsan/tests/unit/tsan_mman_test.cc index 26e13a55c..b2789b70f 100644 --- a/lib/tsan/tests/unit/tsan_mman_test.cc +++ b/lib/tsan/tests/unit/tsan_mman_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mman_test.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mutex_test.cc b/lib/tsan/tests/unit/tsan_mutex_test.cc index cce7f073b..0649c8aa5 100644 --- a/lib/tsan/tests/unit/tsan_mutex_test.cc +++ b/lib/tsan/tests/unit/tsan_mutex_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_mutexset_test.cc b/lib/tsan/tests/unit/tsan_mutexset_test.cc index 335a7748c..5862138a8 100644 --- a/lib/tsan/tests/unit/tsan_mutexset_test.cc +++ b/lib/tsan/tests/unit/tsan_mutexset_test.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset_test.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_shadow_test.cc b/lib/tsan/tests/unit/tsan_shadow_test.cc index 17b17977b..21a4ddf97 100644 --- a/lib/tsan/tests/unit/tsan_shadow_test.cc +++ b/lib/tsan/tests/unit/tsan_shadow_test.cc @@ -1,9 +1,8 @@ //===-- tsan_shadow_test.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_stack_test.cc b/lib/tsan/tests/unit/tsan_stack_test.cc index 92e035d8d..d8b81d331 100644 --- a/lib/tsan/tests/unit/tsan_stack_test.cc +++ b/lib/tsan/tests/unit/tsan_stack_test.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_test.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_sync_test.cc b/lib/tsan/tests/unit/tsan_sync_test.cc index 801665478..7ea2826e8 100644 --- a/lib/tsan/tests/unit/tsan_sync_test.cc +++ b/lib/tsan/tests/unit/tsan_sync_test.cc @@ -1,9 +1,8 @@ //===-- tsan_sync_test.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/tests/unit/tsan_unit_test_main.cc b/lib/tsan/tests/unit/tsan_unit_test_main.cc index 2d55747a0..a1487310f 100644 --- a/lib/tsan/tests/unit/tsan_unit_test_main.cc +++ b/lib/tsan/tests/unit/tsan_unit_test_main.cc @@ -1,9 +1,8 @@ //===-- tsan_unit_test_main.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/CMakeLists.txt b/lib/ubsan/CMakeLists.txt index ab118ae4a..49a3aa121 100644 --- a/lib/ubsan/CMakeLists.txt +++ b/lib/ubsan/CMakeLists.txt @@ -6,18 +6,21 @@ set(UBSAN_SOURCES ubsan_flags.cc ubsan_handlers.cc ubsan_monitor.cc - ubsan_value.cc) + ubsan_value.cc + ) set(UBSAN_STANDALONE_SOURCES ubsan_diag_standalone.cc ubsan_init_standalone.cc - ubsan_signals_standalone.cc) + ubsan_signals_standalone.cc + ) set(UBSAN_CXXABI_SOURCES ubsan_handlers_cxx.cc ubsan_type_hash.cc ubsan_type_hash_itanium.cc - ubsan_type_hash_win.cc) + ubsan_type_hash_win.cc + ) set(UBSAN_HEADERS ubsan_checks.inc @@ -33,7 +36,7 @@ set(UBSAN_HEADERS ubsan_signals_standalone.h ubsan_type_hash.h ubsan_value.h -) + ) include_directories(..) @@ -49,7 +52,7 @@ set(UBSAN_CXXFLAGS ${SANITIZER_COMMON_CFLAGS}) append_rtti_flag(ON UBSAN_CXXFLAGS) append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CXXFLAGS) -set(UBSAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(UBSAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl UBSAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBLOG log UBSAN_DYNAMIC_LIBS) diff --git a/lib/ubsan/ubsan_checks.inc b/lib/ubsan/ubsan_checks.inc index ea82f89e1..7e7216c5b 100644 --- a/lib/ubsan/ubsan_checks.inc +++ b/lib/ubsan/ubsan_checks.inc @@ -1,9 +1,8 @@ //===-- ubsan_checks.inc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_diag.cc b/lib/ubsan/ubsan_diag.cc index df4f13cbe..529cc6985 100644 --- a/lib/ubsan/ubsan_diag.cc +++ b/lib/ubsan/ubsan_diag.cc @@ -1,9 +1,8 @@ //===-- ubsan_diag.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,13 +26,21 @@ using namespace __ubsan; -void __ubsan::GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, - uptr bp, void *context, bool fast) { +// UBSan is combined with runtimes that already provide this functionality +// (e.g., ASan) as well as runtimes that lack it (e.g., scudo). Tried to use +// weak linkage to resolve this issue which is not portable and breaks on +// Windows. +// TODO(yln): This is a temporary workaround. GetStackTrace functions will be +// removed in the future. +void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth, + uptr pc, uptr bp, void *context, bool fast) { uptr top = 0; uptr bottom = 0; - if (fast) + if (StackTrace::WillUseFastUnwind(fast)) { GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(max_depth, pc, bp, context, top, bottom, fast); + stack->Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + stack->Unwind(max_depth, pc, bp, context, 0, 0, false); } static void MaybePrintStackTrace(uptr pc, uptr bp) { @@ -43,7 +50,7 @@ static void MaybePrintStackTrace(uptr pc, uptr bp) { return; BufferedStackTrace stack; - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, + ubsan_GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, common_flags()->fast_unwind_on_fatal); stack.Print(); } diff --git a/lib/ubsan/ubsan_diag.h b/lib/ubsan/ubsan_diag.h index bde749684..b444e971b 100644 --- a/lib/ubsan/ubsan_diag.h +++ b/lib/ubsan/ubsan_diag.h @@ -1,9 +1,8 @@ //===-- ubsan_diag.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,9 +234,6 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET); GET_CALLER_PC_BP; \ ReportOptions Opts = {unrecoverable_handler, pc, bp} -void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, - void *context, bool fast); - /// \brief Instantiate this class before printing diagnostics in the error /// report. This class ensures that reports from different threads and from /// different sanitizers won't be mixed. diff --git a/lib/ubsan/ubsan_diag_standalone.cc b/lib/ubsan/ubsan_diag_standalone.cc index 1f4a5bd40..c22fd1749 100644 --- a/lib/ubsan/ubsan_diag_standalone.cc +++ b/lib/ubsan/ubsan_diag_standalone.cc @@ -1,9 +1,8 @@ //===-- ubsan_diag_standalone.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,20 +16,23 @@ using namespace __ubsan; -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_print_stack_trace() { +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { uptr top = 0; uptr bottom = 0; - bool request_fast_unwind = common_flags()->fast_unwind_on_fatal; - if (request_fast_unwind) - __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, bp, context, 0, 0, false); +} - GET_CURRENT_PC_BP_SP; - (void)sp; +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + GET_CURRENT_PC_BP; BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, - request_fast_unwind); + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal); stack.Print(); } } // extern "C" diff --git a/lib/ubsan/ubsan_flags.cc b/lib/ubsan/ubsan_flags.cc index 7b6784b82..0b87729c3 100644 --- a/lib/ubsan/ubsan_flags.cc +++ b/lib/ubsan/ubsan_flags.cc @@ -1,9 +1,8 @@ //===-- ubsan_flags.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_flags.h b/lib/ubsan/ubsan_flags.h index 18aed9b05..daa0d7c70 100644 --- a/lib/ubsan/ubsan_flags.h +++ b/lib/ubsan/ubsan_flags.h @@ -1,9 +1,8 @@ //===-- ubsan_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_flags.inc b/lib/ubsan/ubsan_flags.inc index e75a4c44e..a4d0e6109 100644 --- a/lib/ubsan/ubsan_flags.inc +++ b/lib/ubsan/ubsan_flags.inc @@ -1,9 +1,8 @@ //===-- ubsan_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_handlers.cc b/lib/ubsan/ubsan_handlers.cc index 11e09b0ff..938ac8975 100644 --- a/lib/ubsan/ubsan_handlers.cc +++ b/lib/ubsan/ubsan_handlers.cc @@ -1,9 +1,8 @@ //===-- ubsan_handlers.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -599,42 +598,6 @@ void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) { Die(); } -static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, - ValueHandle Function, - ReportOptions Opts) { - SourceLocation CallLoc = Data->Loc.acquire(); - ErrorType ET = ErrorType::FunctionTypeMismatch; - - if (ignoreReport(CallLoc, Opts, ET)) - return; - - ScopedReport R(Opts, CallLoc, ET); - - SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); - const char *FName = FLoc.get()->info.function; - if (!FName) - FName = "(unknown)"; - - Diag(CallLoc, DL_Error, ET, - "call to function %0 through pointer to incorrect function type %1") - << FName << Data->Type; - Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; -} - -void -__ubsan::__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, - ValueHandle Function) { - GET_REPORT_OPTIONS(false); - handleFunctionTypeMismatch(Data, Function, Opts); -} - -void __ubsan::__ubsan_handle_function_type_mismatch_abort( - FunctionTypeMismatchData *Data, ValueHandle Function) { - GET_REPORT_OPTIONS(true); - handleFunctionTypeMismatch(Data, Function, Opts); - Die(); -} - static void handleNonNullReturn(NonNullReturnData *Data, SourceLocation *LocPtr, ReportOptions Opts, bool IsAttr) { if (!LocPtr) diff --git a/lib/ubsan/ubsan_handlers.h b/lib/ubsan/ubsan_handlers.h index 2bf9ff432..22ca96422 100644 --- a/lib/ubsan/ubsan_handlers.h +++ b/lib/ubsan/ubsan_handlers.h @@ -1,9 +1,8 @@ //===-- ubsan_handlers.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -169,15 +168,6 @@ struct InvalidBuiltinData { /// Handle a builtin called in an invalid way. RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data) -struct FunctionTypeMismatchData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - -RECOVERABLE(function_type_mismatch, - FunctionTypeMismatchData *Data, - ValueHandle Val) - struct NonNullReturnData { SourceLocation AttrLoc; }; diff --git a/lib/ubsan/ubsan_handlers_cxx.cc b/lib/ubsan/ubsan_handlers_cxx.cc index 85a3e8dad..839bba369 100644 --- a/lib/ubsan/ubsan_handlers_cxx.cc +++ b/lib/ubsan/ubsan_handlers_cxx.cc @@ -1,9 +1,8 @@ //===-- ubsan_handlers_cxx.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -157,6 +156,51 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, Diag(Loc, DL_Note, ET, "check failed in %0, vtable located in %1") << SrcModule << DstModule; } + +static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, + ValueHandle Function, + ValueHandle calleeRTTI, + ValueHandle fnRTTI, ReportOptions Opts) { + if (checkTypeInfoEquality(reinterpret_cast<void *>(calleeRTTI), + reinterpret_cast<void *>(fnRTTI))) + return false; + + SourceLocation CallLoc = Data->Loc.acquire(); + ErrorType ET = ErrorType::FunctionTypeMismatch; + + if (ignoreReport(CallLoc, Opts, ET)) + return true; + + ScopedReport R(Opts, CallLoc, ET); + + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; + + Diag(CallLoc, DL_Error, ET, + "call to function %0 through pointer to incorrect function type %1") + << FName << Data->Type; + Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; + return true; +} + +void __ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, + ValueHandle Function, + ValueHandle calleeRTTI, + ValueHandle fnRTTI) { + GET_REPORT_OPTIONS(false); + handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts); +} + +void __ubsan_handle_function_type_mismatch_abort(FunctionTypeMismatchData *Data, + ValueHandle Function, + ValueHandle calleeRTTI, + ValueHandle fnRTTI) { + GET_REPORT_OPTIONS(true); + if (handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts)) + Die(); +} } // namespace __ubsan #endif // CAN_SANITIZE_UB diff --git a/lib/ubsan/ubsan_handlers_cxx.h b/lib/ubsan/ubsan_handlers_cxx.h index 2ff014edf..be2345dc1 100644 --- a/lib/ubsan/ubsan_handlers_cxx.h +++ b/lib/ubsan/ubsan_handlers_cxx.h @@ -1,9 +1,8 @@ //===-- ubsan_handlers_cxx.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,21 @@ void __ubsan_handle_dynamic_type_cache_miss( extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_dynamic_type_cache_miss_abort( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash); + +struct FunctionTypeMismatchData { + SourceLocation Loc; + const TypeDescriptor &Type; +}; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, + ValueHandle Val, ValueHandle calleeRTTI, + ValueHandle fnRTTI); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch_abort(FunctionTypeMismatchData *Data, + ValueHandle Val, + ValueHandle calleeRTTI, + ValueHandle fnRTTI); } #endif // UBSAN_HANDLERS_H diff --git a/lib/ubsan/ubsan_init.cc b/lib/ubsan/ubsan_init.cc index 32fc434ad..f0bbe1ef1 100644 --- a/lib/ubsan/ubsan_init.cc +++ b/lib/ubsan/ubsan_init.cc @@ -1,9 +1,8 @@ //===-- ubsan_init.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init.h b/lib/ubsan/ubsan_init.h index f12fc2ced..0510385b1 100644 --- a/lib/ubsan/ubsan_init.h +++ b/lib/ubsan/ubsan_init.h @@ -1,9 +1,8 @@ //===-- ubsan_init.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init_standalone.cc b/lib/ubsan/ubsan_init_standalone.cc index 8bd500025..323c2c1f9 100644 --- a/lib/ubsan/ubsan_init_standalone.cc +++ b/lib/ubsan/ubsan_init_standalone.cc @@ -1,9 +1,8 @@ //===-- ubsan_init_standalone.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init_standalone_preinit.cc b/lib/ubsan/ubsan_init_standalone_preinit.cc index 5e75c17ae..bf344a2a9 100644 --- a/lib/ubsan/ubsan_init_standalone_preinit.cc +++ b/lib/ubsan/ubsan_init_standalone_preinit.cc @@ -1,9 +1,8 @@ //===-- ubsan_init_standalone_preinit.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_interface.inc b/lib/ubsan/ubsan_interface.inc index 81e06345d..3eb07b7b9 100644 --- a/lib/ubsan/ubsan_interface.inc +++ b/lib/ubsan/ubsan_interface.inc @@ -1,9 +1,8 @@ //===-- ubsan_interface.inc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Ubsan interface list. diff --git a/lib/ubsan/ubsan_monitor.cc b/lib/ubsan/ubsan_monitor.cc index e2b39845c..cb97a8ff1 100644 --- a/lib/ubsan/ubsan_monitor.cc +++ b/lib/ubsan/ubsan_monitor.cc @@ -1,9 +1,8 @@ //===-- ubsan_monitor.cc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_monitor.h b/lib/ubsan/ubsan_monitor.h index 7159cbb2c..3bfd7be89 100644 --- a/lib/ubsan/ubsan_monitor.h +++ b/lib/ubsan/ubsan_monitor.h @@ -1,9 +1,8 @@ //===-- ubsan_monitor.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_platform.h b/lib/ubsan/ubsan_platform.h index 45a4aa772..71d7fb18c 100644 --- a/lib/ubsan/ubsan_platform.h +++ b/lib/ubsan/ubsan_platform.h @@ -1,9 +1,8 @@ //===-- ubsan_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_signals_standalone.cc b/lib/ubsan/ubsan_signals_standalone.cc index 5e77c60b1..cc7900cb1 100644 --- a/lib/ubsan/ubsan_signals_standalone.cc +++ b/lib/ubsan/ubsan_signals_standalone.cc @@ -1,10 +1,9 @@ //=-- ubsan_signals_standalone.cc //------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,11 +38,15 @@ void InitializeDeadlySignals() {} #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #include "sanitizer_common/sanitizer_signal_interceptors.inc" +// TODO(yln): Temporary workaround. Will be removed. +void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth, + uptr pc, uptr bp, void *context, bool fast); + namespace __ubsan { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + ubsan_GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/ubsan/ubsan_signals_standalone.h b/lib/ubsan/ubsan_signals_standalone.h index b29c29482..128eff266 100644 --- a/lib/ubsan/ubsan_signals_standalone.h +++ b/lib/ubsan/ubsan_signals_standalone.h @@ -1,10 +1,9 @@ //=-- ubsan_signals_standalone.h //------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_type_hash.cc b/lib/ubsan/ubsan_type_hash.cc index a217c862c..431495672 100644 --- a/lib/ubsan/ubsan_type_hash.cc +++ b/lib/ubsan/ubsan_type_hash.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_type_hash.h b/lib/ubsan/ubsan_type_hash.h index aa638713f..e42884b76 100644 --- a/lib/ubsan/ubsan_type_hash.h +++ b/lib/ubsan/ubsan_type_hash.h @@ -1,9 +1,8 @@ //===-- ubsan_type_hash.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -65,6 +64,10 @@ const int VptrMaxOffsetToTop = 1<<20; extern "C" SANITIZER_INTERFACE_ATTRIBUTE HashValue __ubsan_vptr_type_cache[VptrTypeCacheSize]; +/// \brief Do whatever is required by the ABI to check for std::type_info +/// equivalence beyond simple pointer comparison. +bool checkTypeInfoEquality(const void *TypeInfo1, const void *TypeInfo2); + } // namespace __ubsan #endif // UBSAN_TYPE_HASH_H diff --git a/lib/ubsan/ubsan_type_hash_itanium.cc b/lib/ubsan/ubsan_type_hash_itanium.cc index dcce0dd85..c4b048f20 100644 --- a/lib/ubsan/ubsan_type_hash_itanium.cc +++ b/lib/ubsan/ubsan_type_hash_itanium.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash_itanium.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -118,8 +117,7 @@ static bool isDerivedFromAtOffset(const abi::__class_type_info *Derived, const abi::__class_type_info *Base, sptr Offset) { if (Derived->__type_name == Base->__type_name || - (SANITIZER_NON_UNIQUE_TYPEINFO && - !internal_strcmp(Derived->__type_name, Base->__type_name))) + __ubsan::checkTypeInfoEquality(Derived, Base)) return Offset == 0; if (const abi::__si_class_type_info *SI = @@ -258,4 +256,13 @@ __ubsan::getDynamicTypeInfoFromVtable(void *VtablePtr) { ObjectType ? ObjectType->__type_name : "<unknown>"); } +bool __ubsan::checkTypeInfoEquality(const void *TypeInfo1, + const void *TypeInfo2) { + auto TI1 = static_cast<const std::type_info *>(TypeInfo1); + auto TI2 = static_cast<const std::type_info *>(TypeInfo2); + return SANITIZER_NON_UNIQUE_TYPEINFO && TI1->__type_name[0] != '*' && + TI2->__type_name[0] != '*' && + !internal_strcmp(TI1->__type_name, TI2->__type_name); +} + #endif // CAN_SANITIZE_UB && !SANITIZER_WINDOWS diff --git a/lib/ubsan/ubsan_type_hash_win.cc b/lib/ubsan/ubsan_type_hash_win.cc index 271c4aaf6..c7b2e45af 100644 --- a/lib/ubsan/ubsan_type_hash_win.cc +++ b/lib/ubsan/ubsan_type_hash_win.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash_win.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -78,4 +77,8 @@ __ubsan::getDynamicTypeInfoFromVtable(void *VtablePtr) { "<unknown>"); } +bool __ubsan::checkTypeInfoEquality(const void *, const void *) { + return false; +} + #endif // CAN_SANITIZE_UB && SANITIZER_WINDOWS diff --git a/lib/ubsan/ubsan_value.cc b/lib/ubsan/ubsan_value.cc index 466834c09..ba336a667 100644 --- a/lib/ubsan/ubsan_value.cc +++ b/lib/ubsan/ubsan_value.cc @@ -1,9 +1,8 @@ //===-- ubsan_value.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_value.h b/lib/ubsan/ubsan_value.h index 72eee1555..a216e3a14 100644 --- a/lib/ubsan/ubsan_value.h +++ b/lib/ubsan/ubsan_value.h @@ -1,9 +1,8 @@ //===-- ubsan_value.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_dll_thunk.cc b/lib/ubsan/ubsan_win_dll_thunk.cc index a1d0dbd66..fd39e210a 100644 --- a/lib/ubsan/ubsan_win_dll_thunk.cc +++ b/lib/ubsan/ubsan_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_dll_thunk.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc b/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc index c9b74a4c9..87ada6131 100644 --- a/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc +++ b/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_dynamic_runtime_thunk.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_weak_interception.cc b/lib/ubsan/ubsan_win_weak_interception.cc index c28577057..8cf6344ce 100644 --- a/lib/ubsan/ubsan_win_weak_interception.cc +++ b/lib/ubsan/ubsan_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_weak_interception.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Ubsan when it is implemented as a shared diff --git a/lib/xray/tests/CMakeLists.txt b/lib/xray/tests/CMakeLists.txt index 89a2b3b01..a1fbccaeb 100644 --- a/lib/xray/tests/CMakeLists.txt +++ b/lib/xray/tests/CMakeLists.txt @@ -48,8 +48,9 @@ endfunction() set(XRAY_TEST_ARCH ${XRAY_SUPPORTED_ARCH}) set(XRAY_UNITTEST_LINK_FLAGS + ${COMPILER_RT_UNITTEST_LINK_FLAGS} ${CMAKE_THREAD_LIBS_INIT} - -l${SANITIZER_CXX_ABI_LIBRARY}) + ) if (NOT APPLE) # Needed by LLVMSupport. @@ -71,15 +72,20 @@ if (NOT APPLE) endforeach() # We also add the actual libraries to link as dependencies. - list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMXRay -lLLVMSupport -lLLVMTestingSupport) + list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMXRay -lLLVMSupport -lLLVMDemangle -lLLVMTestingSupport) endif() append_list_if(COMPILER_RT_HAS_LIBM -lm XRAY_UNITTEST_LINK_FLAGS) append_list_if(COMPILER_RT_HAS_LIBRT -lrt XRAY_UNITTEST_LINK_FLAGS) append_list_if(COMPILER_RT_HAS_LIBDL -ldl XRAY_UNITTEST_LINK_FLAGS) append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread XRAY_UNITTEST_LINK_FLAGS) + append_list_if(COMPILER_RT_HAS_LIBEXECINFO -lexecinfo XRAY_UNITTEST_LINK_FLAGS) endif() +foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES}) + list(APPEND XRAY_UNITTEST_LINK_FLAGS -l${lib}) +endforeach() + macro(add_xray_unittest testname) cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN}) if(UNIX AND NOT APPLE) diff --git a/lib/xray/tests/unit/allocator_test.cc b/lib/xray/tests/unit/allocator_test.cc index 117074162..d55561335 100644 --- a/lib/xray/tests/unit/allocator_test.cc +++ b/lib/xray/tests/unit/allocator_test.cc @@ -1,9 +1,8 @@ //===-- allocator_test.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/buffer_queue_test.cc b/lib/xray/tests/unit/buffer_queue_test.cc index a30343e18..4af63d095 100644 --- a/lib/xray/tests/unit/buffer_queue_test.cc +++ b/lib/xray/tests/unit/buffer_queue_test.cc @@ -1,9 +1,8 @@ //===-- buffer_queue_test.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/fdr_controller_test.cc b/lib/xray/tests/unit/fdr_controller_test.cc index 8967c4919..7bb87980a 100644 --- a/lib/xray/tests/unit/fdr_controller_test.cc +++ b/lib/xray/tests/unit/fdr_controller_test.cc @@ -1,9 +1,8 @@ //===-- fdr_controller_test.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/fdr_log_writer_test.cc b/lib/xray/tests/unit/fdr_log_writer_test.cc index f2e7a5cba..1ff880a96 100644 --- a/lib/xray/tests/unit/fdr_log_writer_test.cc +++ b/lib/xray/tests/unit/fdr_log_writer_test.cc @@ -1,9 +1,8 @@ //===-- fdr_log_writer_test.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/function_call_trie_test.cc b/lib/xray/tests/unit/function_call_trie_test.cc index 01be69122..6d8df9ae3 100644 --- a/lib/xray/tests/unit/function_call_trie_test.cc +++ b/lib/xray/tests/unit/function_call_trie_test.cc @@ -1,9 +1,8 @@ //===-- function_call_trie_test.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/profile_collector_test.cc b/lib/xray/tests/unit/profile_collector_test.cc index df786d46b..b1bfdc40b 100644 --- a/lib/xray/tests/unit/profile_collector_test.cc +++ b/lib/xray/tests/unit/profile_collector_test.cc @@ -1,9 +1,8 @@ //===-- profile_collector_test.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/test_helpers.cc b/lib/xray/tests/unit/test_helpers.cc index 284492d10..0ed4966ca 100644 --- a/lib/xray/tests/unit/test_helpers.cc +++ b/lib/xray/tests/unit/test_helpers.cc @@ -1,9 +1,8 @@ //===-- test_helpers.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/test_helpers.h b/lib/xray/tests/unit/test_helpers.h index ff0311e9b..62e5831ee 100644 --- a/lib/xray/tests/unit/test_helpers.h +++ b/lib/xray/tests/unit/test_helpers.h @@ -1,9 +1,8 @@ //===-- test_helpers.h ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/tests/unit/xray_unit_test_main.cc b/lib/xray/tests/unit/xray_unit_test_main.cc index 27d17527d..3ab2a623c 100644 --- a/lib/xray/tests/unit/xray_unit_test_main.cc +++ b/lib/xray/tests/unit/xray_unit_test_main.cc @@ -1,9 +1,8 @@ //===-- xray_unit_test_main.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_AArch64.cc b/lib/xray/xray_AArch64.cc index 096de009e..4c7805488 100644 --- a/lib/xray/xray_AArch64.cc +++ b/lib/xray/xray_AArch64.cc @@ -1,9 +1,8 @@ //===-- xray_AArch64.cc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_allocator.h b/lib/xray/xray_allocator.h index 907c54542..4b42c4732 100644 --- a/lib/xray/xray_allocator.h +++ b/lib/xray/xray_allocator.h @@ -1,9 +1,8 @@ //===-- xray_allocator.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_arm.cc b/lib/xray/xray_arm.cc index 5b828287e..db26efaa7 100644 --- a/lib/xray/xray_arm.cc +++ b/lib/xray/xray_arm.cc @@ -1,9 +1,8 @@ //===-- xray_arm.cc ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.cc b/lib/xray/xray_basic_flags.cc index 14d805c71..75b674c85 100644 --- a/lib/xray/xray_basic_flags.cc +++ b/lib/xray/xray_basic_flags.cc @@ -1,9 +1,8 @@ //===-- xray_basic_flags.cc -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.h b/lib/xray/xray_basic_flags.h index 041578f06..2459effa8 100644 --- a/lib/xray/xray_basic_flags.h +++ b/lib/xray/xray_basic_flags.h @@ -1,9 +1,8 @@ //===-- xray_basic_flags.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.inc b/lib/xray/xray_basic_flags.inc index 327735b51..fb38c540d 100644 --- a/lib/xray/xray_basic_flags.inc +++ b/lib/xray/xray_basic_flags.inc @@ -1,9 +1,8 @@ //===-- xray_basic_flags.inc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_logging.cc b/lib/xray/xray_basic_logging.cc index ae1cc0ba7..553041ce0 100644 --- a/lib/xray/xray_basic_logging.cc +++ b/lib/xray/xray_basic_logging.cc @@ -1,9 +1,8 @@ //===-- xray_basic_logging.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_logging.h b/lib/xray/xray_basic_logging.h index 1639b96d9..89caca66b 100644 --- a/lib/xray/xray_basic_logging.h +++ b/lib/xray/xray_basic_logging.h @@ -1,9 +1,8 @@ //===-- xray_basic_logging.h ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_buffer_queue.cc b/lib/xray/xray_buffer_queue.cc index 7d0e5a1f3..4cfa717de 100644 --- a/lib/xray/xray_buffer_queue.cc +++ b/lib/xray/xray_buffer_queue.cc @@ -1,9 +1,8 @@ //===-- xray_buffer_queue.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_buffer_queue.h b/lib/xray/xray_buffer_queue.h index ef2b433f9..e1739d050 100644 --- a/lib/xray/xray_buffer_queue.h +++ b/lib/xray/xray_buffer_queue.h @@ -1,9 +1,8 @@ //===-- xray_buffer_queue.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_defs.h b/lib/xray/xray_defs.h index c009bcc87..2da03c3c3 100644 --- a/lib/xray/xray_defs.h +++ b/lib/xray/xray_defs.h @@ -1,9 +1,8 @@ //===-- xray_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_controller.h b/lib/xray/xray_fdr_controller.h index d44d0309b..28a3546ca 100644 --- a/lib/xray/xray_fdr_controller.h +++ b/lib/xray/xray_fdr_controller.h @@ -1,9 +1,8 @@ //===-- xray_fdr_controller.h ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.cc b/lib/xray/xray_fdr_flags.cc index a14851b1b..8d432d298 100644 --- a/lib/xray/xray_fdr_flags.cc +++ b/lib/xray/xray_fdr_flags.cc @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.h b/lib/xray/xray_fdr_flags.h index 9c953f1ca..d6f00dc48 100644 --- a/lib/xray/xray_fdr_flags.h +++ b/lib/xray/xray_fdr_flags.h @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.inc b/lib/xray/xray_fdr_flags.inc index d8721ad12..6082b7e78 100644 --- a/lib/xray/xray_fdr_flags.inc +++ b/lib/xray/xray_fdr_flags.inc @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.inc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_log_records.h b/lib/xray/xray_fdr_log_records.h index e7b1ee562..7a5d43831 100644 --- a/lib/xray/xray_fdr_log_records.h +++ b/lib/xray/xray_fdr_log_records.h @@ -1,9 +1,8 @@ //===-- xray_fdr_log_records.h -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_log_writer.h b/lib/xray/xray_fdr_log_writer.h index 7712e1377..0378663c3 100644 --- a/lib/xray/xray_fdr_log_writer.h +++ b/lib/xray/xray_fdr_log_writer.h @@ -1,9 +1,8 @@ //===-- xray_fdr_log_writer.h ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_logging.cc b/lib/xray/xray_fdr_logging.cc index 1eda26df7..abba06576 100644 --- a/lib/xray/xray_fdr_logging.cc +++ b/lib/xray/xray_fdr_logging.cc @@ -1,9 +1,8 @@ //===-- xray_fdr_logging.cc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_logging.h b/lib/xray/xray_fdr_logging.h index 1639d550a..6df0057c4 100644 --- a/lib/xray/xray_fdr_logging.h +++ b/lib/xray/xray_fdr_logging.h @@ -1,9 +1,8 @@ //===-- xray_fdr_logging.h ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_flags.cc b/lib/xray/xray_flags.cc index b50b68666..8106a7419 100644 --- a/lib/xray/xray_flags.cc +++ b/lib/xray/xray_flags.cc @@ -1,9 +1,8 @@ //===-- xray_flags.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_flags.h b/lib/xray/xray_flags.h index 7c1ba9458..edb5a5119 100644 --- a/lib/xray/xray_flags.h +++ b/lib/xray/xray_flags.h @@ -1,9 +1,8 @@ //===-- xray_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_flags.inc b/lib/xray/xray_flags.inc index c87903963..b7dc5a08f 100644 --- a/lib/xray/xray_flags.inc +++ b/lib/xray/xray_flags.inc @@ -1,9 +1,8 @@ //===-- xray_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_function_call_trie.h b/lib/xray/xray_function_call_trie.h index d01ad20e3..b8c605837 100644 --- a/lib/xray/xray_function_call_trie.h +++ b/lib/xray/xray_function_call_trie.h @@ -1,9 +1,8 @@ //===-- xray_function_call_trie.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_init.cc b/lib/xray/xray_init.cc index b0922aa8e..b79bc08c5 100644 --- a/lib/xray/xray_init.cc +++ b/lib/xray/xray_init.cc @@ -1,9 +1,8 @@ //===-- xray_init.cc --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_interface.cc b/lib/xray/xray_interface.cc index 6f7b6615b..0d22893eb 100644 --- a/lib/xray/xray_interface.cc +++ b/lib/xray/xray_interface.cc @@ -1,9 +1,8 @@ //===-- xray_interface.cpp --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_interface_internal.h b/lib/xray/xray_interface_internal.h index 8ca874574..0fea63776 100644 --- a/lib/xray/xray_interface_internal.h +++ b/lib/xray/xray_interface_internal.h @@ -1,9 +1,8 @@ //===-- xray_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_log_interface.cc b/lib/xray/xray_log_interface.cc index 0886fd0d1..7916a9e2b 100644 --- a/lib/xray/xray_log_interface.cc +++ b/lib/xray/xray_log_interface.cc @@ -1,9 +1,8 @@ //===-- xray_log_interface.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_mips.cc b/lib/xray/xray_mips.cc index 6f8243828..80990ab8d 100644 --- a/lib/xray/xray_mips.cc +++ b/lib/xray/xray_mips.cc @@ -1,9 +1,8 @@ //===-- xray_mips.cc --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_mips64.cc b/lib/xray/xray_mips64.cc index f1bdf1d7d..73c8924f9 100644 --- a/lib/xray/xray_mips64.cc +++ b/lib/xray/xray_mips64.cc @@ -1,9 +1,8 @@ //===-- xray_mips64.cc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_powerpc64.cc b/lib/xray/xray_powerpc64.cc index 5e4938361..abc2becf5 100644 --- a/lib/xray/xray_powerpc64.cc +++ b/lib/xray/xray_powerpc64.cc @@ -1,9 +1,8 @@ //===-- xray_powerpc64.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_powerpc64.inc b/lib/xray/xray_powerpc64.inc index c1a1bac1a..e4e16d5b2 100644 --- a/lib/xray/xray_powerpc64.inc +++ b/lib/xray/xray_powerpc64.inc @@ -1,9 +1,8 @@ //===-- xray_powerpc64.inc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profile_collector.cc b/lib/xray/xray_profile_collector.cc index dc3a82069..97b52e1d9 100644 --- a/lib/xray/xray_profile_collector.cc +++ b/lib/xray/xray_profile_collector.cc @@ -1,9 +1,8 @@ //===-- xray_profile_collector.cc ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profile_collector.h b/lib/xray/xray_profile_collector.h index 86c4ce853..6e0f25271 100644 --- a/lib/xray/xray_profile_collector.h +++ b/lib/xray/xray_profile_collector.h @@ -1,9 +1,8 @@ //===-- xray_profile_collector.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling.cc b/lib/xray/xray_profiling.cc index 4323170cd..66def6cf2 100644 --- a/lib/xray/xray_profiling.cc +++ b/lib/xray/xray_profiling.cc @@ -1,9 +1,8 @@ //===-- xray_profiling.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.cc b/lib/xray/xray_profiling_flags.cc index 593e66a78..0e89b7420 100644 --- a/lib/xray/xray_profiling_flags.cc +++ b/lib/xray/xray_profiling_flags.cc @@ -1,9 +1,8 @@ //===-- xray_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.h b/lib/xray/xray_profiling_flags.h index 2f9a75147..d67f240ad 100644 --- a/lib/xray/xray_profiling_flags.h +++ b/lib/xray/xray_profiling_flags.h @@ -1,9 +1,8 @@ //===-- xray_profiling_flags.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.inc b/lib/xray/xray_profiling_flags.inc index ccd70860b..4f6138872 100644 --- a/lib/xray/xray_profiling_flags.inc +++ b/lib/xray/xray_profiling_flags.inc @@ -1,9 +1,8 @@ //===-- xray_profiling_flags.inc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_recursion_guard.h b/lib/xray/xray_recursion_guard.h index 6edadea56..3b6158a2d 100644 --- a/lib/xray/xray_recursion_guard.h +++ b/lib/xray/xray_recursion_guard.h @@ -1,9 +1,8 @@ //===-- xray_recursion_guard.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_segmented_array.h b/lib/xray/xray_segmented_array.h index bc7e9379f..6eb673edf 100644 --- a/lib/xray/xray_segmented_array.h +++ b/lib/xray/xray_segmented_array.h @@ -1,9 +1,8 @@ //===-- xray_segmented_array.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_mips.S b/lib/xray/xray_trampoline_mips.S index 39a1a3af3..499c350d2 100644 --- a/lib/xray/xray_trampoline_mips.S +++ b/lib/xray/xray_trampoline_mips.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_mips.s ----------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_mips64.S b/lib/xray/xray_trampoline_mips64.S index 9cbc7e181..d65bec1fc 100644 --- a/lib/xray/xray_trampoline_mips64.S +++ b/lib/xray/xray_trampoline_mips64.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_mips64.s --------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_x86_64.S b/lib/xray/xray_trampoline_x86_64.S index 52985ffd1..1e58362cd 100644 --- a/lib/xray/xray_trampoline_x86_64.S +++ b/lib/xray/xray_trampoline_x86_64.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_tsc.h b/lib/xray/xray_tsc.h index 180d6df18..bd7e1911a 100644 --- a/lib/xray/xray_tsc.h +++ b/lib/xray/xray_tsc.h @@ -1,9 +1,8 @@ //===-- xray_tsc.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_utils.cc b/lib/xray/xray_utils.cc index 59ba6c308..82674baa5 100644 --- a/lib/xray/xray_utils.cc +++ b/lib/xray/xray_utils.cc @@ -1,9 +1,8 @@ //===-- xray_utils.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -79,7 +78,7 @@ void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { // Create VMO to hold the profile data. zx_handle_t Vmo; - zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); + zx_status_t Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo); if (Status != ZX_OK) { Report("XRay: cannot create VMO: %s\n", _zx_status_get_string(Status)); return nullptr; diff --git a/lib/xray/xray_utils.h b/lib/xray/xray_utils.h index 60438973f..333826168 100644 --- a/lib/xray/xray_utils.h +++ b/lib/xray/xray_utils.h @@ -1,9 +1,8 @@ //===-- xray_utils.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_x86_64.inc b/lib/xray/xray_x86_64.inc index b3c475f91..477900355 100644 --- a/lib/xray/xray_x86_64.inc +++ b/lib/xray/xray_x86_64.inc @@ -1,9 +1,8 @@ //===-- xray_x86_64.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // |