[hwasan] Record and display stack history in stack-based reports.

Summary:
Display a list of recent stack frames (not a stack trace!) when
tag-mismatch is detected on a stack address.

The implementation uses alignment tricks to get both the address of
the history buffer, and the base address of the shadow with a single
8-byte load. See the comment in hwasan_thread_list.h for more
details.

Developed in collaboration with Kostya Serebryany.

Reviewers: kcc

Subscribers: srhines, kubamracek, mgorny, hiraditya, jfb, llvm-commits

Differential Revision: https://reviews.llvm.org/D52249

llvm-svn: 342921

compiler-rt/lib/hwasan/CMakeLists.txt

@@ -10,6 +10,7 @@ set(HWASAN_RTL_SOURCES
   hwasan_poisoning.cc
   hwasan_report.cc
   hwasan_thread.cc
+  hwasan_thread_list.cc
   )
 
 set(HWASAN_RTL_CXX_SOURCES
@@ -25,8 +26,9 @@ set(HWASAN_RTL_HEADERS
   hwasan_mapping.h
   hwasan_poisoning.h
   hwasan_report.h
-  hwasan_thread.h)
-
+  hwasan_thread.h
+  hwasan_thread_list.h
+  )
 
 set(HWASAN_DEFINITIONS)
 append_list_if(COMPILER_RT_HWASAN_WITH_INTERCEPTORS HWASAN_WITH_INTERCEPTORS=1 HWASAN_DEFINITIONS)

compiler-rt/lib/hwasan/hwasan.cc

@@ -17,6 +17,7 @@
 #include "hwasan_poisoning.h"
 #include "hwasan_report.h"
 #include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
@@ -174,7 +175,8 @@ static void HWAsanCheckFailed(const char *file, int line, const char *cond,
 static constexpr uptr kMemoryUsageBufferSize = 4096;
 
 static void HwasanFormatMemoryUsage(InternalScopedString &s) {
-  auto thread_stats = Thread::GetThreadStats();
+  HwasanThreadList &thread_list = hwasanThreadList();
+  auto thread_stats = thread_list.GetThreadStats();
   auto *sds = StackDepotGetStats();
   AllocatorStatCounters asc;
   GetAllocatorStats(asc);
@@ -184,7 +186,7 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) {
       " heap: %zd",
       internal_getpid(), GetRSS(), thread_stats.n_live_threads,
       thread_stats.total_stack_size,
-      thread_stats.n_live_threads * Thread::MemoryUsedPerThread(),
+      thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
       sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]);
 }
 
@@ -253,7 +255,12 @@ void __hwasan_init() {
   __sanitizer_set_report_path(common_flags()->log_path);
 
   DisableCoreDumperIfNecessary();
+
   __hwasan_shadow_init();
+
+  InitThreads();
+  hwasanThreadList().CreateCurrentThread();
+
   MadviseShadow();
 
   // This may call libc -> needs initialized shadow.
@@ -268,11 +275,10 @@ void __hwasan_init() {
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 
   HwasanTSDInit();
+  HwasanTSDThreadInit();
 
   HwasanAllocatorInit();
 
-  Thread::Create();
-
 #if HWASAN_CONTAINS_UBSAN
   __ubsan::InitAsPlugin();
 #endif

compiler-rt/lib/hwasan/hwasan.h

@@ -41,6 +41,10 @@ typedef u8 tag_t;
 const unsigned kAddressTagShift = 56;
 const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
 
+// Minimal alignment of the shadow base address. Determines the space available
+// for threads and stack histories. This is an ABI constant.
+const unsigned kShadowBaseAlignment = 32;
+
 static inline tag_t GetTagFromPointer(uptr p) {
   return p >> kAddressTagShift;
 }
@@ -66,6 +70,7 @@ extern int hwasan_report_count;
 
 bool ProtectRange(uptr beg, uptr end);
 bool InitShadow();
+void InitThreads();
 void MadviseShadow();
 char *GetProcSelfMaps();
 void InitializeInterceptors();
@@ -142,6 +147,7 @@ class ScopedThreadLocalStateBackup {
 };
 
 void HwasanTSDInit();
+void HwasanTSDThreadInit();
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context);
 

compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc

@@ -13,6 +13,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "hwasan.h"
 #include "hwasan_dynamic_shadow.h"
 #include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
@@ -35,12 +36,16 @@ static void UnmapFromTo(uptr from, uptr to) {
   }
 }
 
-// Returns an address aligned to 8 pages, such that one page on the left and
-// shadow_size_bytes bytes on the right of it are mapped r/o.
+// Returns an address aligned to kShadowBaseAlignment, such that
+// 2**kShadowBaseAlingment on the left and shadow_size_bytes bytes on the right
+// of it are mapped no access.
 static uptr MapDynamicShadow(uptr shadow_size_bytes) {
   const uptr granularity = GetMmapGranularity();
-  const uptr alignment = granularity << kShadowScale;
-  const uptr left_padding = granularity;
+  const uptr min_alignment = granularity << kShadowScale;
+  const uptr alignment = 1ULL << kShadowBaseAlignment;
+  CHECK_GE(alignment, min_alignment);
+
+  const uptr left_padding = 1ULL << kShadowBaseAlignment;
   const uptr shadow_size =
       RoundUpTo(shadow_size_bytes, granularity);
   const uptr map_size = shadow_size + left_padding + alignment;

compiler-rt/lib/hwasan/hwasan_flags.inc

@@ -51,3 +51,7 @@ HWASAN_FLAG(int, heap_history_size, 1023,
           "to find bugs.")
 HWASAN_FLAG(bool, export_memory_stats, true,
             "Export up-to-date memory stats through /proc")
+HWASAN_FLAG(int, stack_history_size, 1024,
+            "The number of stack frames remembered per thread. "
+            "Affects the quality of stack-related reports, but not the ability "
+            "to find bugs.")

compiler-rt/lib/hwasan/hwasan_linux.cc

@@ -22,6 +22,7 @@
 #include "hwasan_mapping.h"
 #include "hwasan_report.h"
 #include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
 
 #include <elf.h>
 #include <link.h>
@@ -37,6 +38,10 @@
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 
+#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID
+THREADLOCAL uptr __hwasan_tls;
+#endif
+
 namespace __hwasan {
 
 static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
@@ -179,6 +184,20 @@ bool InitShadow() {
   return true;
 }
 
+void InitThreads() {
+  CHECK(__hwasan_shadow_memory_dynamic_address);
+  uptr guard_page_size = GetMmapGranularity();
+  uptr thread_space_start =
+      __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
+  uptr thread_space_end =
+      __hwasan_shadow_memory_dynamic_address - guard_page_size;
+  ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
+                           "hwasan threads");
+  ProtectGap(thread_space_end,
+             __hwasan_shadow_memory_dynamic_address - thread_space_end);
+  InitThreadList(thread_space_start, thread_space_end - thread_space_start);
+}
+
 static void MadviseShadowRegion(uptr beg, uptr end) {
   uptr size = end - beg + 1;
   if (common_flags()->no_huge_pages_for_shadow)
@@ -214,29 +233,33 @@ void InstallAtExitHandler() {
 // ---------------------- TSD ---------------- {{{1
 
 extern "C" void __hwasan_thread_enter() {
-  Thread::Create();
+  hwasanThreadList().CreateCurrentThread();
 }
 
 extern "C" void __hwasan_thread_exit() {
   Thread *t = GetCurrentThread();
   // Make sure that signal handler can not see a stale current thread pointer.
   atomic_signal_fence(memory_order_seq_cst);
   if (t)
-    t->Destroy();
+    hwasanThreadList().ReleaseThread(t);
 }
 
 #if HWASAN_WITH_INTERCEPTORS
 static pthread_key_t tsd_key;
 static bool tsd_key_inited = false;
 
+void HwasanTSDThreadInit() {
+  if (tsd_key_inited)
+    CHECK_EQ(0, pthread_setspecific(tsd_key,
+                                    (void *)GetPthreadDestructorIterations()));
+}
+
 void HwasanTSDDtor(void *tsd) {
-  Thread *t = (Thread*)tsd;
-  if (t->destructor_iterations_ > 1) {
-    t->destructor_iterations_--;
-    CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+  uptr iterations = (uptr)tsd;
+  if (iterations > 1) {
+    CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
     return;
   }
-  t->Destroy();
   __hwasan_thread_exit();
 }
 
@@ -245,31 +268,26 @@ void HwasanTSDInit() {
   tsd_key_inited = true;
   CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
 }
-
-Thread *GetCurrentThread() {
-  return (Thread *)pthread_getspecific(tsd_key);
-}
-
-void SetCurrentThread(Thread *t) {
-  // Make sure that HwasanTSDDtor gets called at the end.
-  CHECK(tsd_key_inited);
-  // Make sure we do not reset the current Thread.
-  CHECK_EQ(0, pthread_getspecific(tsd_key));
-  pthread_setspecific(tsd_key, (void *)t);
-}
-#elif SANITIZER_ANDROID
+#else
 void HwasanTSDInit() {}
-Thread *GetCurrentThread() {
-  return (Thread*)*get_android_tls_ptr();
-}
+void HwasanTSDThreadInit() {}
+#endif
 
-void SetCurrentThread(Thread *t) {
-  *get_android_tls_ptr() = (uptr)t;
+#if SANITIZER_ANDROID
+uptr *GetCurrentThreadLongPtr() {
+  return (uptr *)get_android_tls_ptr();
 }
 #else
-#error unsupported configuration !HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID
+uptr *GetCurrentThreadLongPtr() {
+  return &__hwasan_tls;
+}
 #endif
 
+Thread *GetCurrentThread() {
+  auto *R = (StackAllocationsRingBuffer*)GetCurrentThreadLongPtr();
+  return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next()));
+}
+
 struct AccessInfo {
   uptr addr;
   uptr size;

compiler-rt/lib/hwasan/hwasan_report.cc

@@ -16,6 +16,7 @@
 #include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
@@ -35,6 +36,31 @@ static StackTrace GetStackTraceFromId(u32 id) {
   return res;
 }
 
+// A RAII object that holds a copy of the current thread stack ring buffer.
+// The actual stack buffer may change while we are iterating over it (for
+// example, Printf may call syslog() which can itself be built with hwasan).
+class SavedStackAllocations {
+ public:
+  SavedStackAllocations(StackAllocationsRingBuffer *rb) {
+    uptr size = rb->size() * sizeof(uptr);
+    void *storage =
+        MmapAlignedOrDieOnFatalError(size, size * 2, "saved stack allocations");
+    new (&rb_) StackAllocationsRingBuffer(*rb, storage);
+  }
+
+  ~SavedStackAllocations() {
+    StackAllocationsRingBuffer *rb = get();
+    UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr));
+  }
+
+  StackAllocationsRingBuffer *get() {
+    return (StackAllocationsRingBuffer *)&rb_;
+  }
+
+ private:
+  uptr rb_;
+};
+
 class Decorator: public __sanitizer::SanitizerCommonDecorator {
  public:
   Decorator() : SanitizerCommonDecorator() { }
@@ -63,7 +89,9 @@ uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb,
   return 0;
 }
 
-void PrintAddressDescription(uptr tagged_addr, uptr access_size) {
+void PrintAddressDescription(
+    uptr tagged_addr, uptr access_size,
+    StackAllocationsRingBuffer *current_stack_allocations) {
   Decorator d;
   int num_descriptions_printed = 0;
   uptr untagged_addr = UntagAddr(tagged_addr);
@@ -109,7 +137,7 @@ void PrintAddressDescription(uptr tagged_addr, uptr access_size) {
     }
   }
 
-  Thread::VisitAllLiveThreads([&](Thread *t) {
+  hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
     // Scan all threads' ring buffers to find if it's a heap-use-after-free.
     HeapAllocationRecord har;
     if (uptr D = FindHeapAllocation(t->heap_allocations(), tagged_addr, &har)) {
@@ -145,6 +173,25 @@ void PrintAddressDescription(uptr tagged_addr, uptr access_size) {
       Printf("%s", d.Default());
       t->Announce();
 
+      // Temporary report section, needs to be improved.
+      Printf("Previosly allocated frames:\n");
+      auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+                     ? current_stack_allocations
+                     : t->stack_allocations();
+      uptr frames = Min((uptr)flags()->stack_history_size, sa->size());
+      for (uptr i = 0; i < frames; i++) {
+        uptr record = (*sa)[i];
+        if (!record)
+          break;
+        uptr sp = (record >> 48) << 4;
+        uptr pc_mask = (1ULL << 48) - 1;
+        uptr pc = record & pc_mask;
+        uptr fixed_pc = StackTrace::GetNextInstructionPc(pc);
+        StackTrace stack(&fixed_pc, 1);
+        Printf("record: %p pc: %p sp: %p", record, pc, sp);
+        stack.Print();
+      }
+
       num_descriptions_printed++;
     }
   });
@@ -170,13 +217,16 @@ void ReportStats() {}
 void ReportInvalidAccessInsideAddressRange(const char *what, const void *start,
                                            uptr size, uptr offset) {
   ScopedErrorReportLock l;
+  SavedStackAllocations current_stack_allocations(
+      GetCurrentThread()->stack_allocations());
 
   Decorator d;
   Printf("%s", d.Warning());
   Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n",
          d.Warning(), d.Name(), what, d.Warning(), offset, start, size,
          d.Default());
-  PrintAddressDescription((uptr)start + offset, 1);
+  PrintAddressDescription((uptr)start + offset, 1,
+                          current_stack_allocations.get());
   // if (__sanitizer::Verbosity())
   //   DescribeMemoryRange(start, size);
 }
@@ -224,7 +274,7 @@ void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) {
 
   stack->Print();
 
-  PrintAddressDescription(tagged_addr, 0);
+  PrintAddressDescription(tagged_addr, 0, nullptr);
 
   PrintTagsAroundAddr(tag_ptr);
 
@@ -235,6 +285,8 @@ void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) {
 void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
                        bool is_store) {
   ScopedErrorReportLock l;
+  SavedStackAllocations current_stack_allocations(
+      GetCurrentThread()->stack_allocations());
 
   Decorator d;
   Printf("%s", d.Error());
@@ -258,7 +310,8 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
 
   stack->Print();
 
-  PrintAddressDescription(tagged_addr, access_size);
+  PrintAddressDescription(tagged_addr, access_size,
+                          current_stack_allocations.get());
   t->Announce();
 
   PrintTagsAroundAddr(tag_ptr);