[lsan] Fix deadlock in dl_iterate_phdr.

Summary:
Do not grab the allocator lock before calling dl_iterate_phdr. This may
cause a lock order inversion with (valid) user code that uses malloc
inside a dl_iterate_phdr callback.

Reviewers: vitalybuka, hctim

Subscribers: jfb, #sanitizers, llvm-commits

Tags: #sanitizers, #llvm

Differential Revision: https://reviews.llvm.org/D67738

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@372348 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 52 insertions, 0 deletions

diff --git a/test/lsan/TestCases/Linux/libdl_deadlock.cpp b/test/lsan/TestCases/Linux/libdl_deadlock.cpp
new file mode 100644
index 000000000..7c1f82f10
--- /dev/null
+++ b/test/lsan/TestCases/Linux/libdl_deadlock.cpp
@@ -0,0 +1,52 @@
+// Regression test for a deadlock in leak detection,
+// where lsan would call dl_iterate_phdr while holding the allocator lock.
+// RUN: %clangxx_lsan %s -o %t && %run %t
+
+#include <link.h>
+#include <mutex>
+#include <stdlib.h>
+#include <thread>
+#include <unistd.h>
+
+std::mutex in, out;
+
+int Callback(struct dl_phdr_info *info, size_t size, void *data) {
+  for (int step = 0; step < 50; ++step) {
+    void *p[1000];
+    for (int i = 0; i < 1000; ++i)
+      p[i] = malloc(10 * i);
+
+    if (step == 0)
+      in.unlock();
+
+    for (int i = 0; i < 1000; ++i)
+      free(p[i]);
+  }
+  out.unlock();
+  return 1; // just once
+}
+
+void Watchdog() {
+  // This is just a fail-safe to turn a deadlock (in case the bug reappears)
+  // into a (slow) test failure.
+  usleep(10000000);
+  if (!out.try_lock()) {
+    write(2, "DEADLOCK\n", 9);
+    exit(1);
+  }
+}
+
+int main() {
+  in.lock();
+  out.lock();
+
+  std::thread t([] { dl_iterate_phdr(Callback, nullptr); });
+  t.detach();
+
+  std::thread w(Watchdog);
+  w.detach();
+
+  // Wait for the malloc thread to preheat, then start leak detection (on exit)
+  in.lock();
+  return 0;
+}