This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
[PATCH 3/3] hwasan: add from upstream
Hello.
The last piece is intended for Matthew Malcomson who's been
preparing updated patch set for HWASAN support enablement
and will be installed after his patch set will be approved.
Martin
>From 42bb5c7b4361063a80450d4e699913e055af43cb Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 5 Nov 2019 19:09:00 +0100
Subject: [PATCH 3/3] hwasan: add from upstream
libsanitizer/ChangeLog:
2019-11-05 Martin Liska <mliska@suse.cz>
* merge.sh: Mention hwasan.
* README.gcc: Likewise.
* hwasan/hwasan.cpp: New file.
* hwasan/hwasan.h: New file.
* hwasan/hwasan_allocator.cpp: New file.
* hwasan/hwasan_allocator.h: New file.
* hwasan/hwasan_checks.h: New file.
* hwasan/hwasan_dynamic_shadow.cpp: New file.
* hwasan/hwasan_dynamic_shadow.h: New file.
* hwasan/hwasan_exceptions.cpp: New file.
* hwasan/hwasan_flags.h: New file.
* hwasan/hwasan_flags.inc: New file.
* hwasan/hwasan_interceptors.cpp: New file.
* hwasan/hwasan_interceptors_vfork.S: New file.
* hwasan/hwasan_interface_internal.h: New file.
* hwasan/hwasan_linux.cpp: New file.
* hwasan/hwasan_malloc_bisect.h: New file.
* hwasan/hwasan_mapping.h: New file.
* hwasan/hwasan_memintrinsics.cpp: New file.
* hwasan/hwasan_new_delete.cpp: New file.
* hwasan/hwasan_poisoning.cpp: New file.
* hwasan/hwasan_poisoning.h: New file.
* hwasan/hwasan_report.cpp: New file.
* hwasan/hwasan_report.h: New file.
* hwasan/hwasan_setjmp.S: New file.
* hwasan/hwasan_tag_mismatch_aarch64.S: New file.
* hwasan/hwasan_thread.cpp: New file.
* hwasan/hwasan_thread.h: New file.
* hwasan/hwasan_thread_list.cpp: New file.
* hwasan/hwasan_thread_list.h: New file.
* hwasan/hwasan_type_test.cpp: New file.
---
libsanitizer/README.gcc | 1 +
libsanitizer/hwasan/hwasan.cpp | 616 ++++++++++++++++++
libsanitizer/hwasan/hwasan.h | 195 ++++++
libsanitizer/hwasan/hwasan_allocator.cpp | 406 ++++++++++++
libsanitizer/hwasan/hwasan_allocator.h | 99 +++
libsanitizer/hwasan/hwasan_checks.h | 124 ++++
libsanitizer/hwasan/hwasan_dynamic_shadow.cpp | 164 +++++
libsanitizer/hwasan/hwasan_dynamic_shadow.h | 27 +
libsanitizer/hwasan/hwasan_exceptions.cpp | 67 ++
libsanitizer/hwasan/hwasan_flags.h | 29 +
libsanitizer/hwasan/hwasan_flags.inc | 74 +++
libsanitizer/hwasan/hwasan_interceptors.cpp | 349 ++++++++++
.../hwasan/hwasan_interceptors_vfork.S | 10 +
.../hwasan/hwasan_interface_internal.h | 227 +++++++
libsanitizer/hwasan/hwasan_linux.cpp | 496 ++++++++++++++
libsanitizer/hwasan/hwasan_malloc_bisect.h | 50 ++
libsanitizer/hwasan/hwasan_mapping.h | 57 ++
libsanitizer/hwasan/hwasan_memintrinsics.cpp | 44 ++
libsanitizer/hwasan/hwasan_new_delete.cpp | 66 ++
libsanitizer/hwasan/hwasan_poisoning.cpp | 52 ++
libsanitizer/hwasan/hwasan_poisoning.h | 24 +
libsanitizer/hwasan/hwasan_report.cpp | 572 ++++++++++++++++
libsanitizer/hwasan/hwasan_report.h | 35 +
libsanitizer/hwasan/hwasan_setjmp.S | 100 +++
.../hwasan/hwasan_tag_mismatch_aarch64.S | 152 +++++
libsanitizer/hwasan/hwasan_thread.cpp | 127 ++++
libsanitizer/hwasan/hwasan_thread.h | 112 ++++
libsanitizer/hwasan/hwasan_thread_list.cpp | 15 +
libsanitizer/hwasan/hwasan_thread_list.h | 215 ++++++
libsanitizer/hwasan/hwasan_type_test.cpp | 25 +
libsanitizer/merge.sh | 1 +
31 files changed, 4531 insertions(+)
create mode 100644 libsanitizer/hwasan/hwasan.cpp
create mode 100644 libsanitizer/hwasan/hwasan.h
create mode 100644 libsanitizer/hwasan/hwasan_allocator.cpp
create mode 100644 libsanitizer/hwasan/hwasan_allocator.h
create mode 100644 libsanitizer/hwasan/hwasan_checks.h
create mode 100644 libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
create mode 100644 libsanitizer/hwasan/hwasan_dynamic_shadow.h
create mode 100644 libsanitizer/hwasan/hwasan_exceptions.cpp
create mode 100644 libsanitizer/hwasan/hwasan_flags.h
create mode 100644 libsanitizer/hwasan/hwasan_flags.inc
create mode 100644 libsanitizer/hwasan/hwasan_interceptors.cpp
create mode 100644 libsanitizer/hwasan/hwasan_interceptors_vfork.S
create mode 100644 libsanitizer/hwasan/hwasan_interface_internal.h
create mode 100644 libsanitizer/hwasan/hwasan_linux.cpp
create mode 100644 libsanitizer/hwasan/hwasan_malloc_bisect.h
create mode 100644 libsanitizer/hwasan/hwasan_mapping.h
create mode 100644 libsanitizer/hwasan/hwasan_memintrinsics.cpp
create mode 100644 libsanitizer/hwasan/hwasan_new_delete.cpp
create mode 100644 libsanitizer/hwasan/hwasan_poisoning.cpp
create mode 100644 libsanitizer/hwasan/hwasan_poisoning.h
create mode 100644 libsanitizer/hwasan/hwasan_report.cpp
create mode 100644 libsanitizer/hwasan/hwasan_report.h
create mode 100644 libsanitizer/hwasan/hwasan_setjmp.S
create mode 100644 libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S
create mode 100644 libsanitizer/hwasan/hwasan_thread.cpp
create mode 100644 libsanitizer/hwasan/hwasan_thread.h
create mode 100644 libsanitizer/hwasan/hwasan_thread_list.cpp
create mode 100644 libsanitizer/hwasan/hwasan_thread_list.h
create mode 100644 libsanitizer/hwasan/hwasan_type_test.cpp
diff --git a/libsanitizer/README.gcc b/libsanitizer/README.gcc
index ec491ba4bf8..ffd0403e0af 100644
--- a/libsanitizer/README.gcc
+++ b/libsanitizer/README.gcc
@@ -11,6 +11,7 @@ https://llvm.org/svn/llvm-project/compiler-rt in the following directories:
lib/tsan
lib/lsan
lib/ubsan
+ lib/hwasan
Trivial and urgent fixes (portability, build fixes, etc.) may go directly to the
GCC tree. All non-trivial changes, functionality improvements, etc. should go
diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp
new file mode 100644
index 00000000000..7b5c6c694be
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan.cpp
@@ -0,0 +1,616 @@
+//===-- hwasan.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_checks.h"
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "ubsan/ubsan_flags.h"
+#include "ubsan/ubsan_init.h"
+
+// ACHTUNG! No system header includes in this file.
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+void EnterSymbolizer() {
+ Thread *t = GetCurrentThread();
+ CHECK(t);
+ t->EnterSymbolizer();
+}
+void ExitSymbolizer() {
+ Thread *t = GetCurrentThread();
+ CHECK(t);
+ t->LeaveSymbolizer();
+}
+bool IsInSymbolizer() {
+ Thread *t = GetCurrentThread();
+ return t && t->InSymbolizer();
+}
+
+static Flags hwasan_flags;
+
+Flags *flags() {
+ return &hwasan_flags;
+}
+
+int hwasan_inited = 0;
+int hwasan_instrumentation_inited = 0;
+bool hwasan_init_is_running;
+
+int hwasan_report_count = 0;
+
+void Flags::SetDefaults() {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void InitializeFlags() {
+ SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
+ cf.malloc_context_size = 20;
+ cf.handle_ioctl = true;
+ // FIXME: test and enable.
+ cf.check_printf = false;
+ cf.intercept_tls_get_addr = true;
+ cf.exitcode = 99;
+ // 8 shadow pages ~512kB, small enough to cover common stack sizes.
+ cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
+ // Sigtrap is used in error reporting.
+ cf.handle_sigtrap = kHandleSignalExclusive;
+
+#if SANITIZER_ANDROID
+ // Let platform handle other signals. It is better at reporting them then we
+ // are.
+ cf.handle_segv = kHandleSignalNo;
+ cf.handle_sigbus = kHandleSignalNo;
+ cf.handle_abort = kHandleSignalNo;
+ cf.handle_sigill = kHandleSignalNo;
+ cf.handle_sigfpe = kHandleSignalNo;
+#endif
+ OverrideCommonFlags(cf);
+ }
+
+ Flags *f = flags();
+ f->SetDefaults();
+
+ FlagParser parser;
+ RegisterHwasanFlags(&parser, f);
+ RegisterCommonFlags(&parser);
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::Flags *uf = __ubsan::flags();
+ uf->SetDefaults();
+
+ FlagParser ubsan_parser;
+ __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+ RegisterCommonFlags(&ubsan_parser);
+#endif
+
+ // Override from user-specified string.
+ if (__hwasan_default_options)
+ parser.ParseString(__hwasan_default_options());
+#if HWASAN_CONTAINS_UBSAN
+ const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ ubsan_parser.ParseString(ubsan_default_options);
+#endif
+
+ parser.ParseStringFromEnv("HWASAN_OPTIONS");
+#if HWASAN_CONTAINS_UBSAN
+ ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
+#endif
+
+ InitializeCommonFlags();
+
+ if (Verbosity()) ReportUnrecognizedFlags();
+
+ if (common_flags()->help) parser.PrintFlagDescriptions();
+}
+
+static void HWAsanCheckFailed(const char *file, int line, const char *cond,
+ u64 v1, u64 v2) {
+ Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
+ line, cond, (uptr)v1, (uptr)v2);
+ PRINT_CURRENT_STACK_CHECK();
+ Die();
+}
+
+static constexpr uptr kMemoryUsageBufferSize = 4096;
+
+static void HwasanFormatMemoryUsage(InternalScopedString &s) {
+ HwasanThreadList &thread_list = hwasanThreadList();
+ auto thread_stats = thread_list.GetThreadStats();
+ auto *sds = StackDepotGetStats();
+ AllocatorStatCounters asc;
+ GetAllocatorStats(asc);
+ s.append(
+ "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
+ " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
+ " heap: %zd",
+ internal_getpid(), GetRSS(), thread_stats.n_live_threads,
+ thread_stats.total_stack_size,
+ thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
+ sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]);
+}
+
+#if SANITIZER_ANDROID
+static char *memory_usage_buffer = nullptr;
+
+static void InitMemoryUsage() {
+ memory_usage_buffer =
+ (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string");
+ CHECK(memory_usage_buffer);
+ memory_usage_buffer[0] = '\0';
+ DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize,
+ memory_usage_buffer);
+}
+
+void UpdateMemoryUsage() {
+ if (!flags()->export_memory_stats)
+ return;
+ if (!memory_usage_buffer)
+ InitMemoryUsage();
+ InternalScopedString s(kMemoryUsageBufferSize);
+ HwasanFormatMemoryUsage(s);
+ internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
+ memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
+}
+#else
+void UpdateMemoryUsage() {}
+#endif
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __sanitizer::BufferedStackTrace::UnwindImpl(
+ uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
+ Thread *t = GetCurrentThread();
+ if (!t) {
+ // the thread is still being created.
+ size = 0;
+ return;
+ }
+ if (!StackTrace::WillUseFastUnwind(request_fast)) {
+ // Block reports from our interceptors during _Unwind_Backtrace.
+ SymbolizerScope sym_scope;
+ return Unwind(max_depth, pc, bp, context, 0, 0, request_fast);
+ }
+ if (StackTrace::WillUseFastUnwind(request_fast))
+ Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
+ else
+ Unwind(max_depth, pc, 0, context, 0, 0, false);
+}
+
+struct hwasan_global {
+ s32 gv_relptr;
+ u32 info;
+};
+
+static void InitGlobals(const hwasan_global *begin, const hwasan_global *end) {
+ for (auto *desc = begin; desc != end; ++desc) {
+ uptr gv = reinterpret_cast<uptr>(desc) + desc->gv_relptr;
+ uptr size = desc->info & 0xffffff;
+ uptr full_granule_size = RoundDownTo(size, 16);
+ u8 tag = desc->info >> 24;
+ TagMemoryAligned(gv, full_granule_size, tag);
+ if (size % 16)
+ TagMemoryAligned(gv + full_granule_size, 16, size % 16);
+ }
+}
+
+enum { NT_LLVM_HWASAN_GLOBALS = 3 };
+
+struct hwasan_global_note {
+ s32 begin_relptr;
+ s32 end_relptr;
+};
+
+// Check that the given library meets the code model requirements for tagged
+// globals. These properties are not checked at link time so they need to be
+// checked at runtime.
+static void CheckCodeModel(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum) {
+ ElfW(Addr) min_addr = -1ull, max_addr = 0;
+ for (unsigned i = 0; i != phnum; ++i) {
+ if (phdr[i].p_type != PT_LOAD)
+ continue;
+ ElfW(Addr) lo = base + phdr[i].p_vaddr, hi = lo + phdr[i].p_memsz;
+ if (min_addr > lo)
+ min_addr = lo;
+ if (max_addr < hi)
+ max_addr = hi;
+ }
+
+ if (max_addr - min_addr > 1ull << 32) {
+ Report("FATAL: HWAddressSanitizer: library size exceeds 2^32\n");
+ Die();
+ }
+ if (max_addr > 1ull << 48) {
+ Report("FATAL: HWAddressSanitizer: library loaded above address 2^48\n");
+ Die();
+ }
+}
+
+static void InitGlobalsFromPhdrs(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum) {
+ for (unsigned i = 0; i != phnum; ++i) {
+ if (phdr[i].p_type != PT_NOTE)
+ continue;
+ const char *note = reinterpret_cast<const char *>(base + phdr[i].p_vaddr);
+ const char *nend = note + phdr[i].p_memsz;
+ while (note < nend) {
+ auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(note);
+ const char *name = note + sizeof(ElfW(Nhdr));
+ const char *desc = name + RoundUpTo(nhdr->n_namesz, 4);
+ if (nhdr->n_type != NT_LLVM_HWASAN_GLOBALS ||
+ internal_strcmp(name, "LLVM") != 0) {
+ note = desc + RoundUpTo(nhdr->n_descsz, 4);
+ continue;
+ }
+
+ // Only libraries with instrumented globals need to be checked against the
+ // code model since they use relocations that aren't checked at link time.
+ CheckCodeModel(base, phdr, phnum);
+
+ auto *global_note = reinterpret_cast<const hwasan_global_note *>(desc);
+ auto *global_begin = reinterpret_cast<const hwasan_global *>(
+ note + global_note->begin_relptr);
+ auto *global_end = reinterpret_cast<const hwasan_global *>(
+ note + global_note->end_relptr);
+ InitGlobals(global_begin, global_end);
+ return;
+ }
+ }
+}
+
+static void InitLoadedGlobals() {
+ dl_iterate_phdr(
+ [](dl_phdr_info *info, size_t size, void *data) {
+ InitGlobalsFromPhdrs(info->dlpi_addr, info->dlpi_phdr,
+ info->dlpi_phnum);
+ return 0;
+ },
+ nullptr);
+}
+
+// Prepare to run instrumented code on the main thread.
+static void InitInstrumentation() {
+ if (hwasan_instrumentation_inited) return;
+
+ InitPrctl();
+
+ if (!InitShadow()) {
+ Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
+ DumpProcessMap();
+ Die();
+ }
+
+ InitThreads();
+ hwasanThreadList().CreateCurrentThread();
+
+ hwasan_instrumentation_inited = 1;
+}
+
+// Interface.
+
+uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol.
+
+// This function was used by the old frame descriptor mechanism. We keep it
+// around to avoid breaking ABI.
+void __hwasan_init_frames(uptr beg, uptr end) {}
+
+void __hwasan_init_static() {
+ InitShadowGOT();
+ InitInstrumentation();
+
+ // In the non-static code path we call dl_iterate_phdr here. But at this point
+ // libc might not have been initialized enough for dl_iterate_phdr to work.
+ // Fortunately, since this is a statically linked executable we can use the
+ // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
+ extern ElfW(Ehdr) __ehdr_start;
+ InitGlobalsFromPhdrs(
+ 0,
+ reinterpret_cast<const ElfW(Phdr) *>(
+ reinterpret_cast<const char *>(&__ehdr_start) + __ehdr_start.e_phoff),
+ __ehdr_start.e_phnum);
+}
+
+void __hwasan_init() {
+ CHECK(!hwasan_init_is_running);
+ if (hwasan_inited) return;
+ hwasan_init_is_running = 1;
+ SanitizerToolName = "HWAddressSanitizer";
+
+ InitTlsSize();
+
+ CacheBinaryName();
+ InitializeFlags();
+
+ // Install tool-specific callbacks in sanitizer_common.
+ SetCheckFailedCallback(HWAsanCheckFailed);
+
+ __sanitizer_set_report_path(common_flags()->log_path);
+
+ AndroidTestTlsSlot();
+
+ DisableCoreDumperIfNecessary();
+
+ InitInstrumentation();
+ InitLoadedGlobals();
+
+ // Needs to be called here because flags()->random_tags might not have been
+ // initialized when InitInstrumentation() was called.
+ GetCurrentThread()->InitRandomState();
+
+ MadviseShadow();
+
+ SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
+ // This may call libc -> needs initialized shadow.
+ AndroidLogInit();
+
+ InitializeInterceptors();
+ InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
+ InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+
+ Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+
+ InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
+
+ HwasanTSDInit();
+ HwasanTSDThreadInit();
+
+ HwasanAllocatorInit();
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::InitAsPlugin();
+#endif
+
+ VPrintf(1, "HWAddressSanitizer init done\n");
+
+ hwasan_init_is_running = 0;
+ hwasan_inited = 1;
+}
+
+void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum) {
+ InitGlobalsFromPhdrs(base, phdr, phnum);
+}
+
+void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum) {
+ for (; phnum != 0; ++phdr, --phnum)
+ if (phdr->p_type == PT_LOAD)
+ TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0);
+}
+
+void __hwasan_print_shadow(const void *p, uptr sz) {
+ uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
+ uptr shadow_first = MemToShadow(ptr_raw);
+ uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
+ Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
+ ptr_raw + sz, GetTagFromPointer((uptr)p));
+ for (uptr s = shadow_first; s <= shadow_last; ++s)
+ Printf(" %zx: %x\n", ShadowToMem(s), *(tag_t *)s);
+}
+
+sptr __hwasan_test_shadow(const void *p, uptr sz) {
+ if (sz == 0)
+ return -1;
+ tag_t ptr_tag = GetTagFromPointer((uptr)p);
+ uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
+ uptr shadow_first = MemToShadow(ptr_raw);
+ uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
+ for (uptr s = shadow_first; s <= shadow_last; ++s)
+ if (*(tag_t *)s != ptr_tag) {
+ sptr offset = ShadowToMem(s) - ptr_raw;
+ return offset < 0 ? 0 : offset;
+ }
+ return -1;
+}
+
+u16 __sanitizer_unaligned_load16(const uu16 *p) {
+ return *p;
+}
+u32 __sanitizer_unaligned_load32(const uu32 *p) {
+ return *p;
+}
+u64 __sanitizer_unaligned_load64(const uu64 *p) {
+ return *p;
+}
+void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
+ *p = x;
+}
+
+void __hwasan_loadN(uptr p, uptr sz) {
+ CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
+}
+void __hwasan_load1(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
+}
+void __hwasan_load2(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
+}
+void __hwasan_load4(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
+}
+void __hwasan_load8(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
+}
+void __hwasan_load16(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
+}
+
+void __hwasan_loadN_noabort(uptr p, uptr sz) {
+ CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
+}
+
+void __hwasan_storeN(uptr p, uptr sz) {
+ CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
+}
+void __hwasan_store1(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
+}
+void __hwasan_store2(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
+}
+void __hwasan_store4(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
+}
+void __hwasan_store8(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
+}
+void __hwasan_store16(uptr p) {
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
+}
+
+void __hwasan_storeN_noabort(uptr p, uptr sz) {
+ CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_noabort(uptr p) {
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
+}
+
+void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
+ TagMemoryAligned(p, sz, tag);
+}
+
+uptr __hwasan_tag_pointer(uptr p, u8 tag) {
+ return AddTagToPointer(p, tag);
+}
+
+void __hwasan_handle_longjmp(const void *sp_dst) {
+ uptr dst = (uptr)sp_dst;
+ // HWASan does not support tagged SP.
+ CHECK(GetTagFromPointer(dst) == 0);
+
+ uptr sp = (uptr)__builtin_frame_address(0);
+ static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M
+ if (dst < sp || dst - sp > kMaxExpectedCleanupSize) {
+ Report(
+ "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
+ "stack top: %p; target %p; distance: %p (%zd)\n"
+ "False positive error reports may follow\n",
+ (void *)sp, (void *)dst, dst - sp);
+ return;
+ }
+ TagMemory(sp, dst - sp, 0);
+}
+
+void __hwasan_handle_vfork(const void *sp_dst) {
+ uptr sp = (uptr)sp_dst;
+ Thread *t = GetCurrentThread();
+ CHECK(t);
+ uptr top = t->stack_top();
+ uptr bottom = t->stack_bottom();
+ if (top == 0 || bottom == 0 || sp < bottom || sp >= top) {
+ Report(
+ "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
+ "stack top: %zx; current %zx; bottom: %zx \n"
+ "False positive error reports may follow\n",
+ top, sp, bottom);
+ return;
+ }
+ TagMemory(bottom, sp - bottom, 0);
+}
+
+extern "C" void *__hwasan_extra_spill_area() {
+ Thread *t = GetCurrentThread();
+ return &t->vfork_spill();
+}
+
+void __hwasan_print_memory_usage() {
+ InternalScopedString s(kMemoryUsageBufferSize);
+ HwasanFormatMemoryUsage(s);
+ Printf("%s\n", s.data());
+}
+
+static const u8 kFallbackTag = 0xBB;
+
+u8 __hwasan_generate_tag() {
+ Thread *t = GetCurrentThread();
+ if (!t) return kFallbackTag;
+ return t->GenerateRandomTag();
+}
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char* __hwasan_default_options() { return ""; }
+} // extern "C"
+#endif
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+ stack.Print();
+}
+} // extern "C"
diff --git a/libsanitizer/hwasan/hwasan.h b/libsanitizer/hwasan/hwasan.h
new file mode 100644
index 00000000000..64cdcf30f5c
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan.h
@@ -0,0 +1,195 @@
+//===-- hwasan.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_H
+#define HWASAN_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_flags.h"
+#include "ubsan/ubsan_platform.h"
+
+#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1
+#endif
+
+#ifndef HWASAN_CONTAINS_UBSAN
+# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
+#endif
+
+#ifndef HWASAN_WITH_INTERCEPTORS
+#define HWASAN_WITH_INTERCEPTORS 0
+#endif
+
+typedef u8 tag_t;
+
+// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
+// translation and can be used to store a tag.
+const unsigned kAddressTagShift = 56;
+const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+
+// Minimal alignment of the shadow base address. Determines the space available
+// for threads and stack histories. This is an ABI constant.
+const unsigned kShadowBaseAlignment = 32;
+
+const unsigned kRecordAddrBaseTagShift = 3;
+const unsigned kRecordFPShift = 48;
+const unsigned kRecordFPLShift = 4;
+const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
+
+static inline tag_t GetTagFromPointer(uptr p) {
+ return p >> kAddressTagShift;
+}
+
+static inline uptr UntagAddr(uptr tagged_addr) {
+ return tagged_addr & ~kAddressTagMask;
+}
+
+static inline void *UntagPtr(const void *tagged_ptr) {
+ return reinterpret_cast<void *>(
+ UntagAddr(reinterpret_cast<uptr>(tagged_ptr)));
+}
+
+static inline uptr AddTagToPointer(uptr p, tag_t tag) {
+ return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+}
+
+namespace __hwasan {
+
+extern int hwasan_inited;
+extern bool hwasan_init_is_running;
+extern int hwasan_report_count;
+
+bool ProtectRange(uptr beg, uptr end);
+bool InitShadow();
+void InitPrctl();
+void InitThreads();
+void MadviseShadow();
+char *GetProcSelfMaps();
+void InitializeInterceptors();
+
+void HwasanAllocatorInit();
+void HwasanAllocatorThreadFinish();
+
+void *hwasan_malloc(uptr size, StackTrace *stack);
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack);
+void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack);
+void *hwasan_valloc(uptr size, StackTrace *stack);
+void *hwasan_pvalloc(uptr size, StackTrace *stack);
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack);
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack);
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack);
+void hwasan_free(void *ptr, StackTrace *stack);
+
+void InstallTrapHandler();
+void InstallAtExitHandler();
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+bool IsInSymbolizer();
+
+struct SymbolizerScope {
+ SymbolizerScope() { EnterSymbolizer(); }
+ ~SymbolizerScope() { ExitSymbolizer(); }
+};
+
+// Returns a "chained" origin id, pointing to the given stack trace followed by
+// the previous origin id.
+u32 ChainOrigin(u32 id, StackTrace *stack);
+
+const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
+
+#define GET_MALLOC_STACK_TRACE \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \
+ nullptr, common_flags()->fast_unwind_on_malloc, \
+ common_flags()->malloc_context_size)
+
+#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal)
+
+#define GET_FATAL_STACK_TRACE_HERE \
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
+
+#define PRINT_CURRENT_STACK_CHECK() \
+ { \
+ GET_FATAL_STACK_TRACE_HERE; \
+ stack.Print(); \
+ }
+
+class ScopedThreadLocalStateBackup {
+ public:
+ ScopedThreadLocalStateBackup() { Backup(); }
+ ~ScopedThreadLocalStateBackup() { Restore(); }
+ void Backup();
+ void Restore();
+ private:
+ u64 va_arg_overflow_size_tls;
+};
+
+void HwasanTSDInit();
+void HwasanTSDThreadInit();
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context);
+
+void UpdateMemoryUsage();
+
+void AppendToErrorMessageBuffer(const char *buffer);
+
+void AndroidTestTlsSlot();
+
+} // namespace __hwasan
+
+#define HWASAN_MALLOC_HOOK(ptr, size) \
+ do { \
+ if (&__sanitizer_malloc_hook) { \
+ __sanitizer_malloc_hook(ptr, size); \
+ } \
+ RunMallocHooks(ptr, size); \
+ } while (false)
+#define HWASAN_FREE_HOOK(ptr) \
+ do { \
+ if (&__sanitizer_free_hook) { \
+ __sanitizer_free_hook(ptr); \
+ } \
+ RunFreeHooks(ptr); \
+ } while (false)
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+// For both bionic and glibc __sigset_t is an unsigned long.
+typedef unsigned long __hw_sigset_t;
+// Setjmp and longjmp implementations are platform specific, and hence the
+// interception code is platform specific too. As yet we've only implemented
+// the interception for AArch64.
+typedef unsigned long long __hw_register_buf[22];
+struct __hw_jmp_buf_struct {
+ // NOTE: The machine-dependent definition of `__sigsetjmp'
+ // assume that a `__hw_jmp_buf' begins with a `__hw_register_buf' and that
+ // `__mask_was_saved' follows it. Do not move these members or add others
+ // before it.
+ __hw_register_buf __jmpbuf; // Calling environment.
+ int __mask_was_saved; // Saved the signal mask?
+ __hw_sigset_t __saved_mask; // Saved signal mask.
+};
+typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1];
+typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1];
+#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
+
+#endif // HWASAN_H
diff --git a/libsanitizer/hwasan/hwasan_allocator.cpp b/libsanitizer/hwasan/hwasan_allocator.cpp
new file mode 100644
index 00000000000..81a57d3afd4
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_allocator.cpp
@@ -0,0 +1,406 @@
+//===-- hwasan_allocator.cpp ------------------------ ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_checks.h"
+#include "hwasan_mapping.h"
+#include "hwasan_malloc_bisect.h"
+#include "hwasan_thread.h"
+#include "hwasan_report.h"
+
+namespace __hwasan {
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+static atomic_uint8_t hwasan_allocator_tagging_enabled;
+
+static const tag_t kFallbackAllocTag = 0xBB;
+static const tag_t kFallbackFreeTag = 0xBC;
+
+enum RightAlignMode {
+ kRightAlignNever,
+ kRightAlignSometimes,
+ kRightAlignAlways
+};
+
+// Initialized in HwasanAllocatorInit, an never changed.
+static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1];
+
+bool HwasanChunkView::IsAllocated() const {
+ return metadata_ && metadata_->alloc_context_id && metadata_->requested_size;
+}
+
+// Aligns the 'addr' right to the granule boundary.
+static uptr AlignRight(uptr addr, uptr requested_size) {
+ uptr tail_size = requested_size % kShadowAlignment;
+ if (!tail_size) return addr;
+ return addr + kShadowAlignment - tail_size;
+}
+
+uptr HwasanChunkView::Beg() const {
+ if (metadata_ && metadata_->right_aligned)
+ return AlignRight(block_, metadata_->requested_size);
+ return block_;
+}
+uptr HwasanChunkView::End() const {
+ return Beg() + UsedSize();
+}
+uptr HwasanChunkView::UsedSize() const {
+ return metadata_->requested_size;
+}
+u32 HwasanChunkView::GetAllocStackId() const {
+ return metadata_->alloc_context_id;
+}
+
+uptr HwasanChunkView::ActualSize() const {
+ return allocator.GetActuallyAllocatedSize(reinterpret_cast<void *>(block_));
+}
+
+bool HwasanChunkView::FromSmallHeap() const {
+ return allocator.FromPrimary(reinterpret_cast<void *>(block_));
+}
+
+void GetAllocatorStats(AllocatorStatCounters s) {
+ allocator.GetStats(s);
+}
+
+void HwasanAllocatorInit() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
+ !flags()->disable_allocator_tagging);
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+ for (uptr i = 0; i < sizeof(tail_magic); i++)
+ tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
+}
+
+void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) {
+ allocator.SwallowCache(cache);
+}
+
+static uptr TaggedSize(uptr size) {
+ if (!size) size = 1;
+ uptr new_size = RoundUpTo(size, kShadowAlignment);
+ CHECK_GE(new_size, size);
+ return new_size;
+}
+
+static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
+ bool zeroise) {
+ if (orig_size > kMaxAllowedMallocSize) {
+ if (AllocatorMayReturnNull()) {
+ Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
+ orig_size);
+ return nullptr;
+ }
+ ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack);
+ }
+
+ alignment = Max(alignment, kShadowAlignment);
+ uptr size = TaggedSize(orig_size);
+ Thread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ allocated = allocator.Allocate(t->allocator_cache(), size, alignment);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, size, alignment);
+ }
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportOutOfMemory(size, stack);
+ }
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+ meta->requested_size = static_cast<u32>(orig_size);
+ meta->alloc_context_id = StackDepotPut(*stack);
+ meta->right_aligned = false;
+ if (zeroise) {
+ internal_memset(allocated, 0, size);
+ } else if (flags()->max_malloc_fill_size > 0) {
+ uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size);
+ internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
+ }
+ if (size != orig_size) {
+ internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic,
+ size - orig_size - 1);
+ }
+
+ void *user_ptr = allocated;
+ // Tagging can only be skipped when both tag_in_malloc and tag_in_free are
+ // false. When tag_in_malloc = false and tag_in_free = true malloc needs to
+ // retag to 0.
+ if ((flags()->tag_in_malloc || flags()->tag_in_free) &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
+ if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
+ tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
+ uptr tag_size = orig_size ? orig_size : 1;
+ uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment);
+ user_ptr =
+ (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag);
+ if (full_granule_size != tag_size) {
+ u8 *short_granule =
+ reinterpret_cast<u8 *>(allocated) + full_granule_size;
+ TagMemoryAligned((uptr)short_granule, kShadowAlignment,
+ tag_size % kShadowAlignment);
+ short_granule[kShadowAlignment - 1] = tag;
+ }
+ } else {
+ user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0);
+ }
+ }
+
+ HWASAN_MALLOC_HOOK(user_ptr, size);
+ return user_ptr;
+}
+
+static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
+ CHECK(tagged_ptr);
+ uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
+ tag_t mem_tag = *reinterpret_cast<tag_t *>(
+ MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
+ return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
+}
+
+static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
+ CHECK(tagged_ptr);
+ HWASAN_FREE_HOOK(tagged_ptr);
+
+ if (!PointerAndMemoryTagsMatch(tagged_ptr))
+ ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
+
+ void *untagged_ptr = UntagPtr(tagged_ptr);
+ void *aligned_ptr = reinterpret_cast<void *>(
+ RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
+ uptr orig_size = meta->requested_size;
+ u32 free_context_id = StackDepotPut(*stack);
+ u32 alloc_context_id = meta->alloc_context_id;
+
+ // Check tail magic.
+ uptr tagged_size = TaggedSize(orig_size);
+ if (flags()->free_checks_tail_magic && orig_size &&
+ tagged_size != orig_size) {
+ uptr tail_size = tagged_size - orig_size - 1;
+ CHECK_LT(tail_size, kShadowAlignment);
+ void *tail_beg = reinterpret_cast<void *>(
+ reinterpret_cast<uptr>(aligned_ptr) + orig_size);
+ if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size))
+ ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr),
+ orig_size, tail_magic);
+ }
+
+ meta->requested_size = 0;
+ meta->alloc_context_id = 0;
+ // This memory will not be reused by anyone else, so we are free to keep it
+ // poisoned.
+ Thread *t = GetCurrentThread();
+ if (flags()->max_free_fill_size > 0) {
+ uptr fill_size =
+ Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
+ internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
+ }
+ if (flags()->tag_in_free && malloc_bisect(stack, 0) &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
+ t ? t->GenerateRandomTag() : kFallbackFreeTag);
+ if (t) {
+ allocator.Deallocate(t->allocator_cache(), aligned_ptr);
+ if (auto *ha = t->heap_allocations())
+ ha->push({reinterpret_cast<uptr>(tagged_ptr), alloc_context_id,
+ free_context_id, static_cast<u32>(orig_size)});
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, aligned_ptr);
+ }
+}
+
+static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
+ uptr new_size, uptr alignment) {
+ if (!PointerAndMemoryTagsMatch(tagged_ptr_old))
+ ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr_old));
+
+ void *tagged_ptr_new =
+ HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
+ if (tagged_ptr_old && tagged_ptr_new) {
+ void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
+ internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old,
+ Min(new_size, static_cast<uptr>(meta->requested_size)));
+ HwasanDeallocate(stack, tagged_ptr_old);
+ }
+ return tagged_ptr_new;
+}
+
+static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportCallocOverflow(nmemb, size, stack);
+ }
+ return HwasanAllocate(stack, nmemb * size, sizeof(u64), true);
+}
+
+HwasanChunkView FindHeapChunkByAddress(uptr address) {
+ void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
+ if (!block)
+ return HwasanChunkView();
+ Metadata *metadata =
+ reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
+ return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
+static uptr AllocationSize(const void *tagged_ptr) {
+ const void *untagged_ptr = UntagPtr(tagged_ptr);
+ if (!untagged_ptr) return 0;
+ const void *beg = allocator.GetBlockBegin(untagged_ptr);
+ Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
+ if (b->right_aligned) {
+ if (beg != reinterpret_cast<void *>(RoundDownTo(
+ reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)))
+ return 0;
+ } else {
+ if (beg != untagged_ptr) return 0;
+ }
+ return b->requested_size;
+}
+
+void *hwasan_malloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+}
+
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));
+}
+
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
+ if (!ptr)
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+ if (size == 0) {
+ HwasanDeallocate(stack, ptr);
+ return nullptr;
+ }
+ return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
+}
+
+void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportReallocArrayOverflow(nmemb, size, stack);
+ }
+ return hwasan_realloc(ptr, nmemb * size, stack);
+}
+
+void *hwasan_valloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(
+ HwasanAllocate(stack, size, GetPageSizeCached(), false));
+}
+
+void *hwasan_pvalloc(uptr size, StackTrace *stack) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportPvallocOverflow(size, stack);
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
+}
+
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAlignedAllocAlignment(size, alignment, stack);
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAllocationAlignment(alignment, stack);
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ if (AllocatorMayReturnNull())
+ return errno_EINVAL;
+ ReportInvalidPosixMemalignAlignment(alignment, stack);
+ }
+ void *ptr = HwasanAllocate(stack, size, alignment, false);
+ if (UNLIKELY(!ptr))
+ // OOM error is already taken care of by HwasanAllocate.
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+void hwasan_free(void *ptr, StackTrace *stack) {
+ return HwasanDeallocate(stack, ptr);
+}
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __hwasan_enable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
+}
+
+void __hwasan_disable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
+}
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h
new file mode 100644
index 00000000000..f62be269602
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_allocator.h
@@ -0,0 +1,99 @@
+//===-- hwasan_allocator.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_ALLOCATOR_H
+#define HWASAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_ring_buffer.h"
+#include "hwasan_poisoning.h"
+
+#if !defined(__aarch64__) && !defined(__x86_64__)
+#error Unsupported platform
+#endif
+
+namespace __hwasan {
+
+struct Metadata {
+ u32 requested_size : 31; // sizes are < 2G.
+ u32 right_aligned : 1;
+ u32 alloc_context_id;
+};
+
+struct HwasanMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); }
+ void OnUnmap(uptr p, uptr size) const {
+ // We are about to unmap a chunk of user memory.
+ // It can return as user-requested mmap() or another thread stack.
+ // Make it accessible with zero-tagged pointer.
+ TagMemory(p, size, 0);
+ }
+};
+
+static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+
+struct AP64 {
+ static const uptr kSpaceBeg = ~0ULL;
+ static const uptr kSpaceSize = 0x2000000000ULL;
+ static const uptr kMetadataSize = sizeof(Metadata);
+ typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap;
+ using AddressSpaceView = LocalAddressSpaceView;
+ typedef HwasanMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator> Allocator;
+typedef Allocator::AllocatorCache AllocatorCache;
+
+void AllocatorSwallowThreadLocalCache(AllocatorCache *cache);
+
+class HwasanChunkView {
+ public:
+ HwasanChunkView() : block_(0), metadata_(nullptr) {}
+ HwasanChunkView(uptr block, Metadata *metadata)
+ : block_(block), metadata_(metadata) {}
+ bool IsAllocated() const; // Checks if the memory is currently allocated
+ uptr Beg() const; // First byte of user memory
+ uptr End() const; // Last byte of user memory
+ uptr UsedSize() const; // Size requested by the user
+ uptr ActualSize() const; // Size allocated by the allocator.
+ u32 GetAllocStackId() const;
+ bool FromSmallHeap() const;
+ private:
+ uptr block_;
+ Metadata *const metadata_;
+};
+
+HwasanChunkView FindHeapChunkByAddress(uptr address);
+
+// Information about one (de)allocation that happened in the past.
+// These are recorded in a thread-local ring buffer.
+// TODO: this is currently 24 bytes (20 bytes + alignment).
+// Compress it to 16 bytes or extend it to be more useful.
+struct HeapAllocationRecord {
+ uptr tagged_addr;
+ u32 alloc_context_id;
+ u32 free_context_id;
+ u32 requested_size;
+};
+
+typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
+
+void GetAllocatorStats(AllocatorStatCounters s);
+
+} // namespace __hwasan
+
+#endif // HWASAN_ALLOCATOR_H
diff --git a/libsanitizer/hwasan/hwasan_checks.h b/libsanitizer/hwasan/hwasan_checks.h
new file mode 100644
index 00000000000..a8de0fef20f
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_checks.h
@@ -0,0 +1,124 @@
+//===-- hwasan_checks.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_CHECKS_H
+#define HWASAN_CHECKS_H
+
+#include "hwasan_mapping.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+template <unsigned X>
+__attribute__((always_inline)) static void SigTrap(uptr p) {
+#if defined(__aarch64__)
+ (void)p;
+ // 0x900 is added to do not interfere with the kernel use of lower values of
+ // brk immediate.
+ register uptr x0 asm("x0") = p;
+ asm("brk %1\n\t" ::"r"(x0), "n"(0x900 + X));
+#elif defined(__x86_64__)
+ // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes
+ // total. The pointer is passed via rdi.
+ // 0x40 is added as a safeguard, to help distinguish our trap from others and
+ // to avoid 0 offsets in the command (otherwise it'll be reduced to a
+ // different nop command, the three bytes one).
+ asm volatile(
+ "int3\n"
+ "nopl %c0(%%rax)\n" ::"n"(0x40 + X),
+ "D"(p));
+#else
+ // FIXME: not always sigill.
+ __builtin_trap();
+#endif
+ // __builtin_unreachable();
+}
+
+// Version with access size which is not power of 2
+template <unsigned X>
+__attribute__((always_inline)) static void SigTrap(uptr p, uptr size) {
+#if defined(__aarch64__)
+ register uptr x0 asm("x0") = p;
+ register uptr x1 asm("x1") = size;
+ asm("brk %2\n\t" ::"r"(x0), "r"(x1), "n"(0x900 + X));
+#elif defined(__x86_64__)
+ // Size is stored in rsi.
+ asm volatile(
+ "int3\n"
+ "nopl %c0(%%rax)\n" ::"n"(0x40 + X),
+ "D"(p), "S"(size));
+#else
+ __builtin_trap();
+#endif
+ // __builtin_unreachable();
+}
+
+__attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches(
+ tag_t mem_tag, uptr ptr, uptr sz) {
+ tag_t ptr_tag = GetTagFromPointer(ptr);
+ if (ptr_tag == mem_tag)
+ return true;
+ if (mem_tag >= kShadowAlignment)
+ return false;
+ if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag)
+ return false;
+#ifndef __aarch64__
+ ptr = UntagAddr(ptr);
+#endif
+ return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag;
+}
+
+enum class ErrorAction { Abort, Recover };
+enum class AccessType { Load, Store };
+
+template <ErrorAction EA, AccessType AT, unsigned LogSize>
+__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw);
+ if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) {
+ SigTrap<0x20 * (EA == ErrorAction::Recover) +
+ 0x10 * (AT == AccessType::Store) + LogSize>(p);
+ if (EA == ErrorAction::Abort)
+ __builtin_unreachable();
+ }
+}
+
+template <ErrorAction EA, AccessType AT>
+__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
+ uptr sz) {
+ if (sz == 0)
+ return;
+ tag_t ptr_tag = GetTagFromPointer(p);
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw);
+ tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz);
+ for (tag_t *t = shadow_first; t < shadow_last; ++t)
+ if (UNLIKELY(ptr_tag != *t)) {
+ SigTrap<0x20 * (EA == ErrorAction::Recover) +
+ 0x10 * (AT == AccessType::Store) + 0xf>(p, sz);
+ if (EA == ErrorAction::Abort)
+ __builtin_unreachable();
+ }
+ uptr end = p + sz;
+ uptr tail_sz = end & 0xf;
+ if (UNLIKELY(tail_sz != 0 &&
+ !PossiblyShortTagMatches(
+ *shadow_last, end & ~(kShadowAlignment - 1), tail_sz))) {
+ SigTrap<0x20 * (EA == ErrorAction::Recover) +
+ 0x10 * (AT == AccessType::Store) + 0xf>(p, sz);
+ if (EA == ErrorAction::Abort)
+ __builtin_unreachable();
+ }
+}
+
+} // end namespace __hwasan
+
+#endif // HWASAN_CHECKS_H
diff --git a/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
new file mode 100644
index 00000000000..a04751f44a3
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
@@ -0,0 +1,164 @@
+//===-- hwasan_dynamic_shadow.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region and handles ifunc resolver case, when necessary.
+///
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_mapping.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
+#include <elf.h>
+#include <link.h>
+
+// The code in this file needs to run in an unrelocated binary. It should not
+// access any external symbol, including its own non-hidden globals.
+
+namespace __hwasan {
+
+static void UnmapFromTo(uptr from, uptr to) {
+ if (to == from)
+ return;
+ CHECK(to >= from);
+ uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
+ if (UNLIKELY(internal_iserror(res))) {
+ Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
+ SanitizerToolName, to - from, to - from, from);
+ CHECK("unable to unmap" && 0);
+ }
+}
+
+// Returns an address aligned to kShadowBaseAlignment, such that
+// 2**kShadowBaseAlingment on the left and shadow_size_bytes bytes on the right
+// of it are mapped no access.
+static uptr MapDynamicShadow(uptr shadow_size_bytes) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr min_alignment = granularity << kShadowScale;
+ const uptr alignment = 1ULL << kShadowBaseAlignment;
+ CHECK_GE(alignment, min_alignment);
+
+ const uptr left_padding = 1ULL << kShadowBaseAlignment;
+ const uptr shadow_size =
+ RoundUpTo(shadow_size_bytes, granularity);
+ const uptr map_size = shadow_size + left_padding + alignment;
+
+ const uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+
+ UnmapFromTo(map_start, shadow_start - left_padding);
+ UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+ return shadow_start;
+}
+
+} // namespace __hwasan
+
+#if SANITIZER_ANDROID
+extern "C" {
+
+INTERFACE_ATTRIBUTE void __hwasan_shadow();
+decltype(__hwasan_shadow)* __hwasan_premap_shadow();
+
+} // extern "C"
+
+namespace __hwasan {
+
+// Conservative upper limit.
+static uptr PremapShadowSize() {
+ return RoundUpTo(GetMaxVirtualAddress() >> kShadowScale,
+ GetMmapGranularity());
+}
+
+static uptr PremapShadow() {
+ return MapDynamicShadow(PremapShadowSize());
+}
+
+static bool IsPremapShadowAvailable() {
+ const uptr shadow = reinterpret_cast<uptr>(&__hwasan_shadow);
+ const uptr resolver = reinterpret_cast<uptr>(&__hwasan_premap_shadow);
+ // shadow == resolver is how Android KitKat and older handles ifunc.
+ // shadow == 0 just in case.
+ return shadow != 0 && shadow != resolver;
+}
+
+static uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr shadow_start = reinterpret_cast<uptr>(&__hwasan_shadow);
+ const uptr premap_shadow_size = PremapShadowSize();
+ const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
+
+ // We may have mapped too much. Release extra memory.
+ UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
+ return shadow_start;
+}
+
+} // namespace __hwasan
+
+extern "C" {
+
+decltype(__hwasan_shadow)* __hwasan_premap_shadow() {
+ // The resolver might be called multiple times. Map the shadow just once.
+ static __sanitizer::uptr shadow = 0;
+ if (!shadow)
+ shadow = __hwasan::PremapShadow();
+ return reinterpret_cast<decltype(__hwasan_shadow)*>(shadow);
+}
+
+// __hwasan_shadow is a "function" that has the same address as the first byte
+// of the shadow mapping.
+INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow")))
+void __hwasan_shadow();
+
+extern __attribute((weak, visibility("hidden"))) ElfW(Rela) __rela_iplt_start[],
+ __rela_iplt_end[];
+
+} // extern "C"
+
+namespace __hwasan {
+
+void InitShadowGOT() {
+ // Call the ifunc resolver for __hwasan_shadow and fill in its GOT entry. This
+ // needs to be done before other ifunc resolvers (which are handled by libc)
+ // because a resolver might read __hwasan_shadow.
+ typedef ElfW(Addr) (*ifunc_resolver_t)(void);
+ for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) {
+ ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset);
+ ElfW(Addr) resolver = r->r_addend;
+ if (resolver == reinterpret_cast<ElfW(Addr)>(&__hwasan_premap_shadow)) {
+ *offset = reinterpret_cast<ifunc_resolver_t>(resolver)();
+ break;
+ }
+ }
+}
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+ if (IsPremapShadowAvailable())
+ return FindPremappedShadowStart(shadow_size_bytes);
+ return MapDynamicShadow(shadow_size_bytes);
+}
+
+} // namespace __hwasan
+#else
+namespace __hwasan {
+
+void InitShadowGOT() {}
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+ return MapDynamicShadow(shadow_size_bytes);
+}
+
+} // namespace __hwasan
+
+#endif // SANITIZER_ANDROID
diff --git a/libsanitizer/hwasan/hwasan_dynamic_shadow.h b/libsanitizer/hwasan/hwasan_dynamic_shadow.h
new file mode 100644
index 00000000000..3c2e7c716a3
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_dynamic_shadow.h
@@ -0,0 +1,27 @@
+//===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_PREMAP_SHADOW_H
+#define HWASAN_PREMAP_SHADOW_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __hwasan {
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes);
+void InitShadowGOT();
+
+} // namespace __hwasan
+
+#endif // HWASAN_PREMAP_SHADOW_H
diff --git a/libsanitizer/hwasan/hwasan_exceptions.cpp b/libsanitizer/hwasan/hwasan_exceptions.cpp
new file mode 100644
index 00000000000..169e7876cb5
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_exceptions.cpp
@@ -0,0 +1,67 @@
+//===-- hwasan_exceptions.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+#include <unwind.h>
+
+using namespace __hwasan;
+using namespace __sanitizer;
+
+typedef _Unwind_Reason_Code PersonalityFn(int version, _Unwind_Action actions,
+ uint64_t exception_class,
+ _Unwind_Exception* unwind_exception,
+ _Unwind_Context* context);
+
+// Pointers to the _Unwind_GetGR and _Unwind_GetCFA functions are passed in
+// instead of being called directly. This is to handle cases where the unwinder
+// is statically linked and the sanitizer runtime and the program are linked
+// against different unwinders. The _Unwind_Context data structure is opaque so
+// it may be incompatible between unwinders.
+typedef _Unwind_Word GetGRFn(_Unwind_Context* context, int index);
+typedef _Unwind_Word GetCFAFn(_Unwind_Context* context);
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE _Unwind_Reason_Code
+__hwasan_personality_wrapper(int version, _Unwind_Action actions,
+ uint64_t exception_class,
+ _Unwind_Exception* unwind_exception,
+ _Unwind_Context* context,
+ PersonalityFn* real_personality, GetGRFn* get_gr,
+ GetCFAFn* get_cfa) {
+ _Unwind_Reason_Code rc;
+ if (real_personality)
+ rc = real_personality(version, actions, exception_class, unwind_exception,
+ context);
+ else
+ rc = _URC_CONTINUE_UNWIND;
+
+ // We only untag frames without a landing pad because landing pads are
+ // responsible for untagging the stack themselves if they resume.
+ //
+ // Here we assume that the frame record appears after any locals. This is not
+ // required by AAPCS but is a requirement for HWASAN instrumented functions.
+ if ((actions & _UA_CLEANUP_PHASE) && rc == _URC_CONTINUE_UNWIND) {
+#if defined(__x86_64__)
+ uptr fp = get_gr(context, 6); // rbp
+#elif defined(__aarch64__)
+ uptr fp = get_gr(context, 29); // x29
+#else
+#error Unsupported architecture
+#endif
+ uptr sp = get_cfa(context);
+ TagMemory(sp, fp - sp, 0);
+ }
+
+ return rc;
+}
diff --git a/libsanitizer/hwasan/hwasan_flags.h b/libsanitizer/hwasan/hwasan_flags.h
new file mode 100644
index 00000000000..0a6998f675d
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_flags.h
@@ -0,0 +1,29 @@
+//===-- hwasan_flags.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAGS_H
+#define HWASAN_FLAGS_H
+
+namespace __hwasan {
+
+struct Flags {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+
+ void SetDefaults();
+};
+
+Flags *flags();
+
+} // namespace __hwasan
+
+#endif // HWASAN_FLAGS_H
diff --git a/libsanitizer/hwasan/hwasan_flags.inc b/libsanitizer/hwasan/hwasan_flags.inc
new file mode 100644
index 00000000000..dffbf56cb15
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_flags.inc
@@ -0,0 +1,74 @@
+//===-- hwasan_flags.inc ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Hwasan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAG
+# error "Define HWASAN_FLAG prior to including this file!"
+#endif
+
+// HWASAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+HWASAN_FLAG(bool, verbose_threads, false,
+ "inform on thread creation/destruction")
+HWASAN_FLAG(bool, tag_in_malloc, true, "")
+HWASAN_FLAG(bool, tag_in_free, true, "")
+HWASAN_FLAG(bool, print_stats, false, "")
+HWASAN_FLAG(bool, halt_on_error, true, "")
+HWASAN_FLAG(bool, atexit, false, "")
+
+// Test only flag to disable malloc/realloc/free memory tagging on startup.
+// Tagging can be reenabled with __hwasan_enable_allocator_tagging().
+HWASAN_FLAG(bool, disable_allocator_tagging, false, "")
+
+// If false, use simple increment of a thread local counter to generate new
+// tags.
+HWASAN_FLAG(bool, random_tags, true, "")
+
+HWASAN_FLAG(
+ int, max_malloc_fill_size, 0x1000, // By default, fill only the first 4K.
+ "HWASan allocator flag. max_malloc_fill_size is the maximal amount of "
+ "bytes that will be filled with malloc_fill_byte on malloc.")
+
+HWASAN_FLAG(bool, free_checks_tail_magic, 1,
+ "If set, free() will check the magic values "
+ "to the right of the allocated object "
+ "if the allocation size is not a divident of the granule size")
+HWASAN_FLAG(
+ int, max_free_fill_size, 0,
+ "HWASan allocator flag. max_free_fill_size is the maximal amount of "
+ "bytes that will be filled with free_fill_byte during free.")
+HWASAN_FLAG(int, malloc_fill_byte, 0xbe,
+ "Value used to fill the newly allocated memory.")
+HWASAN_FLAG(int, free_fill_byte, 0x55,
+ "Value used to fill deallocated memory.")
+HWASAN_FLAG(int, heap_history_size, 1023,
+ "The number of heap (de)allocations remembered per thread. "
+ "Affects the quality of heap-related reports, but not the ability "
+ "to find bugs.")
+HWASAN_FLAG(bool, export_memory_stats, true,
+ "Export up-to-date memory stats through /proc")
+HWASAN_FLAG(int, stack_history_size, 1024,
+ "The number of stack frames remembered per thread. "
+ "Affects the quality of stack-related reports, but not the ability "
+ "to find bugs.")
+
+// Malloc / free bisection. Only tag malloc and free calls when a hash of
+// allocation size and stack trace is between malloc_bisect_left and
+// malloc_bisect_right (both inclusive). [0, 0] range is special and disables
+// bisection (i.e. everything is tagged). Once the range is narrowed down
+// enough, use malloc_bisect_dump to see interesting allocations.
+HWASAN_FLAG(uptr, malloc_bisect_left, 0,
+ "Left bound of malloc bisection, inclusive.")
+HWASAN_FLAG(uptr, malloc_bisect_right, 0,
+ "Right bound of malloc bisection, inclusive.")
+HWASAN_FLAG(bool, malloc_bisect_dump, false,
+ "Print all allocations within [malloc_bisect_left, "
+ "malloc_bisect_right] range ")
diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp
new file mode 100644
index 00000000000..44e569ee6d7
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_interceptors.cpp
@@ -0,0 +1,349 @@
+//===-- hwasan_interceptors.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for standard library functions.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.h
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_report.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+#include <stdarg.h>
+// ACHTUNG! No other system header includes in this file.
+// Ideally, we should get rid of stdarg.h as well.
+
+using namespace __hwasan;
+
+using __sanitizer::memory_order;
+using __sanitizer::atomic_load;
+using __sanitizer::atomic_store;
+using __sanitizer::atomic_uintptr_t;
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+#define ENSURE_HWASAN_INITED() do { \
+ CHECK(!hwasan_init_is_running); \
+ if (!hwasan_inited) { \
+ __hwasan_init(); \
+ } \
+} while (0)
+
+
+int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ CHECK_NE(memptr, 0);
+ int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+ return res;
+}
+
+void * __sanitizer_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_memalign(alignment, size, &stack);
+}
+
+void * __sanitizer_aligned_alloc(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+void * __sanitizer___libc_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ void *ptr = hwasan_memalign(alignment, size, &stack);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+void * __sanitizer_valloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_valloc(size, &stack);
+}
+
+void * __sanitizer_pvalloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_pvalloc(size, &stack);
+}
+
+void __sanitizer_free(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ hwasan_free(ptr, &stack);
+}
+
+void __sanitizer_cfree(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ hwasan_free(ptr, &stack);
+}
+
+uptr __sanitizer_malloc_usable_size(const void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
+ __sanitizer_struct_mallinfo sret;
+ internal_memset(&sret, 0, sizeof(sret));
+ return sret;
+}
+
+int __sanitizer_mallopt(int cmd, int value) {
+ return 0;
+}
+
+void __sanitizer_malloc_stats(void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+
+void * __sanitizer_calloc(uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return hwasan_calloc(nmemb, size, &stack);
+}
+
+void * __sanitizer_realloc(void *ptr, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!hwasan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = hwasan_malloc(copy_size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return hwasan_realloc(ptr, size, &stack);
+}
+
+void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_reallocarray(ptr, nmemb, size, &stack);
+}
+
+void * __sanitizer_malloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_init_is_running))
+ ENSURE_HWASAN_INITED();
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return hwasan_malloc(size, &stack);
+}
+
+#if HWASAN_WITH_INTERCEPTORS
+#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
+ ALIAS("__sanitizer_" #FN); \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
+ ARGS) ALIAS("__sanitizer_" #FN)
+
+INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
+ SIZE_T size);
+INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, free, void *ptr);
+INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
+INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, cfree, void *ptr);
+INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
+INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
+INTERCEPTOR_ALIAS(void, malloc_stats, void);
+#endif
+
+struct ThreadStartArg {
+ thread_callback_t callback;
+ void *param;
+};
+
+static void *HwasanThreadStartFunc(void *arg) {
+ __hwasan_thread_enter();
+ ThreadStartArg A = *reinterpret_cast<ThreadStartArg*>(arg);
+ UnmapOrDie(arg, GetPageSizeCached());
+ return A.callback(A.param);
+}
+
+INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
+ void * param) {
+ ScopedTaggingDisabler disabler;
+ ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
+ GetPageSizeCached(), "pthread_create"));
+ *A = {callback, param};
+ int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr),
+ &HwasanThreadStartFunc, A);
+ return res;
+}
+
+DEFINE_REAL(int, vfork)
+DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
+#endif // HWASAN_WITH_INTERCEPTORS
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+// Get and/or change the set of blocked signals.
+extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set,
+ __hw_sigset_t *__restrict __oset);
+#define SIG_BLOCK 0
+#define SIG_SETMASK 2
+extern "C" int __sigjmp_save(__hw_sigjmp_buf env, int savemask) {
+ env[0].__mask_was_saved =
+ (savemask && sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0,
+ &env[0].__saved_mask) == 0);
+ return 0;
+}
+
+static void __attribute__((always_inline))
+InternalLongjmp(__hw_register_buf env, int retval) {
+ // Clear all memory tags on the stack between here and where we're going.
+ unsigned long long stack_pointer = env[13];
+ // The stack pointer should never be tagged, so we don't need to clear the
+ // tag for this function call.
+ __hwasan_handle_longjmp((void *)stack_pointer);
+
+ // Run code for handling a longjmp.
+ // Need to use a register that isn't going to be loaded from the environment
+ // buffer -- hence why we need to specify the register to use.
+ // Must implement this ourselves, since we don't know the order of registers
+ // in different libc implementations and many implementations mangle the
+ // stack pointer so we can't use it without knowing the demangling scheme.
+ register long int retval_tmp asm("x1") = retval;
+ register void *env_address asm("x0") = &env[0];
+ asm volatile("ldp x19, x20, [%0, #0<<3];"
+ "ldp x21, x22, [%0, #2<<3];"
+ "ldp x23, x24, [%0, #4<<3];"
+ "ldp x25, x26, [%0, #6<<3];"
+ "ldp x27, x28, [%0, #8<<3];"
+ "ldp x29, x30, [%0, #10<<3];"
+ "ldp d8, d9, [%0, #14<<3];"
+ "ldp d10, d11, [%0, #16<<3];"
+ "ldp d12, d13, [%0, #18<<3];"
+ "ldp d14, d15, [%0, #20<<3];"
+ "ldr x5, [%0, #13<<3];"
+ "mov sp, x5;"
+ // Return the value requested to return through arguments.
+ // This should be in x1 given what we requested above.
+ "cmp %1, #0;"
+ "mov x0, #1;"
+ "csel x0, %1, x0, ne;"
+ "br x30;"
+ : "+r"(env_address)
+ : "r"(retval_tmp));
+}
+
+INTERCEPTOR(void, siglongjmp, __hw_sigjmp_buf env, int val) {
+ if (env[0].__mask_was_saved)
+ // Restore the saved signal mask.
+ (void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask,
+ (__hw_sigset_t *)0);
+ InternalLongjmp(env[0].__jmpbuf, val);
+}
+
+// Required since glibc libpthread calls __libc_longjmp on pthread_exit, and
+// _setjmp on start_thread. Hence we have to intercept the longjmp on
+// pthread_exit so the __hw_jmp_buf order matches.
+INTERCEPTOR(void, __libc_longjmp, __hw_jmp_buf env, int val) {
+ InternalLongjmp(env[0].__jmpbuf, val);
+}
+
+INTERCEPTOR(void, longjmp, __hw_jmp_buf env, int val) {
+ InternalLongjmp(env[0].__jmpbuf, val);
+}
+#undef SIG_BLOCK
+#undef SIG_SETMASK
+
+#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
+
+static void BeforeFork() {
+ StackDepotLockAll();
+}
+
+static void AfterFork() {
+ StackDepotUnlockAll();
+}
+
+INTERCEPTOR(int, fork, void) {
+ ENSURE_HWASAN_INITED();
+ BeforeFork();
+ int pid = REAL(fork)();
+ AfterFork();
+ return pid;
+}
+
+namespace __hwasan {
+
+int OnExit() {
+ // FIXME: ask frontend whether we need to return failure.
+ return 0;
+}
+
+} // namespace __hwasan
+
+namespace __hwasan {
+
+void InitializeInterceptors() {
+ static int inited = 0;
+ CHECK_EQ(inited, 0);
+
+ INTERCEPT_FUNCTION(fork);
+
+#if HWASAN_WITH_INTERCEPTORS
+#if defined(__linux__)
+ INTERCEPT_FUNCTION(vfork);
+#endif // __linux__
+ INTERCEPT_FUNCTION(pthread_create);
+#endif
+
+ inited = 1;
+}
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_interceptors_vfork.S b/libsanitizer/hwasan/hwasan_interceptors_vfork.S
new file mode 100644
index 00000000000..13d0829c086
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_interceptors_vfork.S
@@ -0,0 +1,10 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS
+#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area
+#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
+#endif
+
+NO_EXEC_STACK_DIRECTIVE
diff --git a/libsanitizer/hwasan/hwasan_interface_internal.h b/libsanitizer/hwasan/hwasan_interface_internal.h
new file mode 100644
index 00000000000..aedda317497
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_interface_internal.h
@@ -0,0 +1,227 @@
+//===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan interface header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_INTERFACE_INTERNAL_H
+#define HWASAN_INTERFACE_INTERNAL_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include <link.h>
+
+extern "C" {
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init_static();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+ ElfW(Half) phnum);
+
+using __sanitizer::uptr;
+using __sanitizer::sptr;
+using __sanitizer::uu64;
+using __sanitizer::uu32;
+using __sanitizer::uu16;
+using __sanitizer::u64;
+using __sanitizer::u32;
+using __sanitizer::u16;
+using __sanitizer::u8;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init_frames(uptr, uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+extern uptr __hwasan_shadow_memory_dynamic_address;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN_noabort(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_noabort(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN_noabort(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_noabort(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_tag_memory(uptr p, u8 tag, uptr sz);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __hwasan_tag_pointer(uptr p, u8 tag);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_tag_mismatch(uptr addr, u8 ts);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u8 __hwasan_generate_tag();
+
+// Returns the offset of the first tag mismatch or -1 if the whole range is
+// good.
+SANITIZER_INTERFACE_ATTRIBUTE
+sptr __hwasan_test_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+/* OPTIONAL */ const char* __hwasan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_print_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_handle_longjmp(const void *sp_dst);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_handle_vfork(const void *sp_dst);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *p, u16 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *p, u32 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *p, u64 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_enable_allocator_tagging();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_disable_allocator_tagging();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_thread_enter();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_thread_exit();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_print_memory_usage();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_memalign(uptr alignment, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_aligned_alloc(uptr alignment, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer___libc_memalign(uptr alignment, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_valloc(uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_pvalloc(uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_free(void *ptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_cfree(void *ptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __sanitizer_malloc_usable_size(const void *ptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+__hwasan::__sanitizer_struct_mallinfo __sanitizer_mallinfo();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_mallopt(int cmd, int value);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_malloc_stats(void);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_calloc(uptr nmemb, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_realloc(void *ptr, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void * __sanitizer_malloc(uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memcpy(void *dst, const void *src, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memset(void *s, int c, uptr n);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memmove(void *dest, const void *src, uptr n);
+} // extern "C"
+
+#endif // HWASAN_INTERFACE_INTERNAL_H
diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp
new file mode 100644
index 00000000000..ed0f30161b0
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_linux.cpp
@@ -0,0 +1,496 @@
+//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
+/// FreeBSD-specific code.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+
+#include "hwasan.h"
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+
+#include <dlfcn.h>
+#include <elf.h>
+#include <link.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/resource.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <sys/prctl.h>
+#include <errno.h>
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
+//
+// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
+// Not currently tested.
+// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
+// Integration tests downstream exist.
+// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
+// Tested with check-hwasan on x86_64-linux.
+// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
+// Tested with check-hwasan on aarch64-linux-android.
+#if !SANITIZER_ANDROID
+SANITIZER_INTERFACE_ATTRIBUTE
+THREADLOCAL uptr __hwasan_tls;
+#endif
+
+namespace __hwasan {
+
+static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+ CHECK_EQ((beg % GetMmapGranularity()), 0);
+ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+ uptr size = end - beg + 1;
+ DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
+ if (!MmapFixedNoReserve(beg, size, name)) {
+ Report(
+ "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+ "Perhaps you're using ulimit -v\n",
+ size);
+ Abort();
+ }
+}
+
+static void ProtectGap(uptr addr, uptr size) {
+ if (!size)
+ return;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ // A few pages at the start of the address space can not be protected.
+ // But we really want to protect as much as possible, to prevent this memory
+ // being returned as a result of a non-FIXED mmap().
+ if (addr == 0) {
+ uptr step = GetMmapGranularity();
+ while (size > step) {
+ addr += step;
+ size -= step;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ }
+ }
+
+ Report(
+ "ERROR: Failed to protect shadow gap [%p, %p]. "
+ "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
+ (void *)(addr + size));
+ DumpProcessMap();
+ Die();
+}
+
+static uptr kLowMemStart;
+static uptr kLowMemEnd;
+static uptr kLowShadowEnd;
+static uptr kLowShadowStart;
+static uptr kHighShadowStart;
+static uptr kHighShadowEnd;
+static uptr kHighMemStart;
+static uptr kHighMemEnd;
+
+static void PrintRange(uptr start, uptr end, const char *name) {
+ Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
+}
+
+static void PrintAddressSpaceLayout() {
+ PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
+ if (kHighShadowEnd + 1 < kHighMemStart)
+ PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
+ PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
+ if (kLowShadowEnd + 1 < kHighShadowStart)
+ PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
+ PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
+ if (kLowMemEnd + 1 < kLowShadowStart)
+ PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
+ PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
+ CHECK_EQ(0, kLowMemStart);
+}
+
+static uptr GetHighMemEnd() {
+ // HighMem covers the upper part of the address space.
+ uptr max_address = GetMaxUserVirtualAddress();
+ // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
+ // properly aligned:
+ max_address |= (GetMmapGranularity() << kShadowScale) - 1;
+ return max_address;
+}
+
+static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
+ __hwasan_shadow_memory_dynamic_address =
+ FindDynamicShadowStart(shadow_size_bytes);
+}
+
+void InitPrctl() {
+#define PR_SET_TAGGED_ADDR_CTRL 55
+#define PR_GET_TAGGED_ADDR_CTRL 56
+#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+ // Check we're running on a kernel that can use the tagged address ABI.
+ if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) == (uptr)-1 &&
+ errno == EINVAL) {
+#if SANITIZER_ANDROID
+ // Some older Android kernels have the tagged pointer ABI on
+ // unconditionally, and hence don't have the tagged-addr prctl while still
+ // allow the ABI.
+ // If targeting Android and the prctl is not around we assume this is the
+ // case.
+ return;
+#else
+ Printf(
+ "FATAL: "
+ "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
+ Die();
+#endif
+ }
+
+ // Turn on the tagged address ABI.
+ if (internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) ==
+ (uptr)-1 ||
+ !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) {
+ Printf(
+ "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+ "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+ "configuration.\n");
+ Die();
+ }
+#undef PR_SET_TAGGED_ADDR_CTRL
+#undef PR_GET_TAGGED_ADDR_CTRL
+#undef PR_TAGGED_ADDR_ENABLE
+}
+
+bool InitShadow() {
+ // Define the entire memory range.
+ kHighMemEnd = GetHighMemEnd();
+
+ // Determine shadow memory base offset.
+ InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
+
+ // Place the low memory first.
+ kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
+ kLowMemStart = 0;
+
+ // Define the low shadow based on the already placed low memory.
+ kLowShadowEnd = MemToShadow(kLowMemEnd);
+ kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
+
+ // High shadow takes whatever memory is left up there (making sure it is not
+ // interfering with low memory in the fixed case).
+ kHighShadowEnd = MemToShadow(kHighMemEnd);
+ kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
+
+ // High memory starts where allocated shadow allows.
+ kHighMemStart = ShadowToMem(kHighShadowStart);
+
+ // Check the sanity of the defined memory ranges (there might be gaps).
+ CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
+ CHECK_GT(kHighMemStart, kHighShadowEnd);
+ CHECK_GT(kHighShadowEnd, kHighShadowStart);
+ CHECK_GT(kHighShadowStart, kLowMemEnd);
+ CHECK_GT(kLowMemEnd, kLowMemStart);
+ CHECK_GT(kLowShadowEnd, kLowShadowStart);
+ CHECK_GT(kLowShadowStart, kLowMemEnd);
+
+ if (Verbosity())
+ PrintAddressSpaceLayout();
+
+ // Reserve shadow memory.
+ ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
+ ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
+
+ // Protect all the gaps.
+ ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
+ if (kLowMemEnd + 1 < kLowShadowStart)
+ ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
+ if (kLowShadowEnd + 1 < kHighShadowStart)
+ ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
+ if (kHighShadowEnd + 1 < kHighMemStart)
+ ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
+
+ return true;
+}
+
+void InitThreads() {
+ CHECK(__hwasan_shadow_memory_dynamic_address);
+ uptr guard_page_size = GetMmapGranularity();
+ uptr thread_space_start =
+ __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
+ uptr thread_space_end =
+ __hwasan_shadow_memory_dynamic_address - guard_page_size;
+ ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
+ "hwasan threads");
+ ProtectGap(thread_space_end,
+ __hwasan_shadow_memory_dynamic_address - thread_space_end);
+ InitThreadList(thread_space_start, thread_space_end - thread_space_start);
+}
+
+static void MadviseShadowRegion(uptr beg, uptr end) {
+ uptr size = end - beg + 1;
+ SetShadowRegionHugePageMode(beg, size);
+ if (common_flags()->use_madv_dontdump)
+ DontDumpShadowMemory(beg, size);
+}
+
+void MadviseShadow() {
+ MadviseShadowRegion(kLowShadowStart, kLowShadowEnd);
+ MadviseShadowRegion(kHighShadowStart, kHighShadowEnd);
+}
+
+bool MemIsApp(uptr p) {
+ CHECK(GetTagFromPointer(p) == 0);
+ return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
+}
+
+static void HwasanAtExit(void) {
+ if (common_flags()->print_module_map)
+ DumpProcessMap();
+ if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+ ReportStats();
+ if (hwasan_report_count > 0) {
+ // ReportAtExitStatistics();
+ if (common_flags()->exitcode)
+ internal__exit(common_flags()->exitcode);
+ }
+}
+
+void InstallAtExitHandler() {
+ atexit(HwasanAtExit);
+}
+
+// ---------------------- TSD ---------------- {{{1
+
+extern "C" void __hwasan_thread_enter() {
+ hwasanThreadList().CreateCurrentThread()->InitRandomState();
+}
+
+extern "C" void __hwasan_thread_exit() {
+ Thread *t = GetCurrentThread();
+ // Make sure that signal handler can not see a stale current thread pointer.
+ atomic_signal_fence(memory_order_seq_cst);
+ if (t)
+ hwasanThreadList().ReleaseThread(t);
+}
+
+#if HWASAN_WITH_INTERCEPTORS
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+
+void HwasanTSDThreadInit() {
+ if (tsd_key_inited)
+ CHECK_EQ(0, pthread_setspecific(tsd_key,
+ (void *)GetPthreadDestructorIterations()));
+}
+
+void HwasanTSDDtor(void *tsd) {
+ uptr iterations = (uptr)tsd;
+ if (iterations > 1) {
+ CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
+ return;
+ }
+ __hwasan_thread_exit();
+}
+
+void HwasanTSDInit() {
+ CHECK(!tsd_key_inited);
+ tsd_key_inited = true;
+ CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
+}
+#else
+void HwasanTSDInit() {}
+void HwasanTSDThreadInit() {}
+#endif
+
+#if SANITIZER_ANDROID
+uptr *GetCurrentThreadLongPtr() {
+ return (uptr *)get_android_tls_ptr();
+}
+#else
+uptr *GetCurrentThreadLongPtr() {
+ return &__hwasan_tls;
+}
+#endif
+
+#if SANITIZER_ANDROID
+void AndroidTestTlsSlot() {
+ uptr kMagicValue = 0x010203040A0B0C0D;
+ uptr *tls_ptr = GetCurrentThreadLongPtr();
+ uptr old_value = *tls_ptr;
+ *tls_ptr = kMagicValue;
+ dlerror();
+ if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
+ Printf(
+ "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
+ "for dlerror().\n");
+ Die();
+ }
+ *tls_ptr = old_value;
+}
+#else
+void AndroidTestTlsSlot() {}
+#endif
+
+Thread *GetCurrentThread() {
+ auto *R = (StackAllocationsRingBuffer *)GetCurrentThreadLongPtr();
+ return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next()));
+}
+
+struct AccessInfo {
+ uptr addr;
+ uptr size;
+ bool is_store;
+ bool is_load;
+ bool recover;
+};
+
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+ // Access type is passed in a platform dependent way (see below) and encoded
+ // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
+ // recoverable. Valid values of Y are 0 to 4, which are interpreted as
+ // log2(access_size), and 0xF, which means that access size is passed via
+ // platform dependent register (see below).
+#if defined(__aarch64__)
+ // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
+ // access size is stored in X1 register. Access address is always in X0
+ // register.
+ uptr pc = (uptr)info->si_addr;
+ const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
+ if ((code & 0xff00) != 0x900)
+ return AccessInfo{}; // Not ours.
+
+ const bool is_store = code & 0x10;
+ const bool recover = code & 0x20;
+ const uptr addr = uc->uc_mcontext.regs[0];
+ const unsigned size_log = code & 0xf;
+ if (size_log > 4 && size_log != 0xf)
+ return AccessInfo{}; // Not ours.
+ const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
+
+#elif defined(__x86_64__)
+ // Access type is encoded in the instruction following INT3 as
+ // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
+ // RSI register. Access address is always in RDI register.
+ uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
+ uint8_t *nop = (uint8_t*)pc;
+ if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
+ *(nop + 3) < 0x40)
+ return AccessInfo{}; // Not ours.
+ const unsigned code = *(nop + 3);
+
+ const bool is_store = code & 0x10;
+ const bool recover = code & 0x20;
+ const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
+ const unsigned size_log = code & 0xf;
+ if (size_log > 4 && size_log != 0xf)
+ return AccessInfo{}; // Not ours.
+ const uptr size =
+ size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
+
+#else
+# error Unsupported architecture
+#endif
+
+ return AccessInfo{addr, size, is_store, !is_store, recover};
+}
+
+static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame,
+ ucontext_t *uc, uptr *registers_frame = nullptr) {
+ InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
+
+ // The second stack frame contains the failure __hwasan_check function, as
+ // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
+ // we wish to ignore. This (currently) only occurs on AArch64, as x64
+ // implementations use SIGTRAP to implement the failure, and thus do not go
+ // through the stack saver.
+ if (registers_frame && stack->trace && stack->size > 0) {
+ stack->trace++;
+ stack->size--;
+ }
+
+ bool fatal = flags()->halt_on_error || !ai.recover;
+ ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
+ registers_frame);
+}
+
+static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
+ AccessInfo ai = GetAccessInfo(info, uc);
+ if (!ai.is_store && !ai.is_load)
+ return false;
+
+ SignalContext sig{info, uc};
+ HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
+
+#if defined(__aarch64__)
+ uc->uc_mcontext.pc += 4;
+#elif defined(__x86_64__)
+#else
+# error Unsupported architecture
+#endif
+ return true;
+}
+
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {
+ // Probably a tag mismatch.
+ if (signo == SIGTRAP)
+ if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
+ return;
+
+ HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+
+} // namespace __hwasan
+
+// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
+// rest of the mismatch handling code (C++).
+void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize) {
+ __hwasan::AccessInfo ai;
+ ai.is_store = access_info & 0x10;
+ ai.is_load = !ai.is_store;
+ ai.recover = access_info & 0x20;
+ ai.addr = addr;
+ if ((access_info & 0xf) == 0xf)
+ ai.size = outsize;
+ else
+ ai.size = 1 << (access_info & 0xf);
+
+ __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
+ (uptr)__builtin_frame_address(0), nullptr,
+ registers_frame);
+ __builtin_unreachable();
+}
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/hwasan/hwasan_malloc_bisect.h b/libsanitizer/hwasan/hwasan_malloc_bisect.h
new file mode 100644
index 00000000000..eaf124aab7d
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_malloc_bisect.h
@@ -0,0 +1,50 @@
+//===-- hwasan_malloc_bisect.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_hash.h"
+#include "hwasan.h"
+
+namespace __hwasan {
+
+static u32 malloc_hash(StackTrace *stack, uptr orig_size) {
+ uptr len = Min(stack->size, (unsigned)7);
+ MurMur2HashBuilder H(len);
+ H.add(orig_size);
+ // Start with frame #1 to skip __sanitizer_malloc frame, which is
+ // (a) almost always the same (well, could be operator new or new[])
+ // (b) can change hashes when compiler-rt is rebuilt, invalidating previous
+ // bisection results.
+ // Because of ASLR, use only offset inside the page.
+ for (uptr i = 1; i < len; ++i) H.add(((u32)stack->trace[i]) & 0xFFF);
+ return H.get();
+}
+
+static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) {
+ uptr left = flags()->malloc_bisect_left;
+ uptr right = flags()->malloc_bisect_right;
+ if (LIKELY(left == 0 && right == 0))
+ return true;
+ if (!stack)
+ return true;
+ // Allow malloc_bisect_right > (u32)(-1) to avoid spelling the latter in
+ // decimal.
+ uptr h = (uptr)malloc_hash(stack, orig_size);
+ if (h < left || h > right)
+ return false;
+ if (flags()->malloc_bisect_dump) {
+ Printf("[alloc] %u %zu\n", h, orig_size);
+ stack->Print();
+ }
+ return true;
+}
+
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_mapping.h b/libsanitizer/hwasan/hwasan_mapping.h
new file mode 100644
index 00000000000..a86ad7ca803
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_mapping.h
@@ -0,0 +1,57 @@
+//===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and defines memory mapping.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_MAPPING_H
+#define HWASAN_MAPPING_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "hwasan_interface_internal.h"
+
+// Typical mapping on Linux/x86_64:
+// with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]:
+// || [0x7f0d59f40000, 0x7fffffffffff] || HighMem ||
+// || [0x7efe2f934000, 0x7f0d59f3ffff] || HighShadow ||
+// || [0x7e7e2f934000, 0x7efe2f933fff] || ShadowGap ||
+// || [0x770d59f40000, 0x7e7e2f933fff] || LowShadow ||
+// || [0x000000000000, 0x770d59f3ffff] || LowMem ||
+
+// Typical mapping on Android/AArch64
+// with dynamic shadow mapped: [0x007477480000, 0x007c77480000]:
+// || [0x007c77480000, 0x007fffffffff] || HighMem ||
+// || [0x007c3ebc8000, 0x007c7747ffff] || HighShadow ||
+// || [0x007bbebc8000, 0x007c3ebc7fff] || ShadowGap ||
+// || [0x007477480000, 0x007bbebc7fff] || LowShadow ||
+// || [0x000000000000, 0x00747747ffff] || LowMem ||
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+constexpr uptr kShadowScale = 4;
+constexpr uptr kShadowAlignment = 1ULL << kShadowScale;
+
+namespace __hwasan {
+
+inline uptr MemToShadow(uptr untagged_addr) {
+ return (untagged_addr >> kShadowScale) +
+ __hwasan_shadow_memory_dynamic_address;
+}
+inline uptr ShadowToMem(uptr shadow_addr) {
+ return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale;
+}
+inline uptr MemToShadowSize(uptr size) {
+ return size >> kShadowScale;
+}
+
+bool MemIsApp(uptr p);
+
+} // namespace __hwasan
+
+#endif // HWASAN_MAPPING_H
diff --git a/libsanitizer/hwasan/hwasan_memintrinsics.cpp b/libsanitizer/hwasan/hwasan_memintrinsics.cpp
new file mode 100644
index 00000000000..e82d77a1bc1
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_memintrinsics.cpp
@@ -0,0 +1,44 @@
+//===-- hwasan_memintrinsics.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains HWASAN versions of
+/// memset, memcpy and memmove
+///
+//===----------------------------------------------------------------------===//
+
+#include <string.h>
+#include "hwasan.h"
+#include "hwasan_checks.h"
+#include "hwasan_flags.h"
+#include "hwasan_interface_internal.h"
+#include "sanitizer_common/sanitizer_libc.h"
+
+using namespace __hwasan;
+
+void *__hwasan_memset(void *block, int c, uptr size) {
+ CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+ reinterpret_cast<uptr>(block), size);
+ return memset(UntagPtr(block), c, size);
+}
+
+void *__hwasan_memcpy(void *to, const void *from, uptr size) {
+ CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+ reinterpret_cast<uptr>(to), size);
+ CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
+ reinterpret_cast<uptr>(from), size);
+ return memcpy(UntagPtr(to), UntagPtr(from), size);
+}
+
+void *__hwasan_memmove(void *to, const void *from, uptr size) {
+ CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+ reinterpret_cast<uptr>(to), size);
+ CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
+ reinterpret_cast<uptr>(from), size);
+ return memmove(UntagPtr(to), UntagPtr(from), size);
+}
diff --git a/libsanitizer/hwasan/hwasan_new_delete.cpp b/libsanitizer/hwasan/hwasan_new_delete.cpp
new file mode 100644
index 00000000000..191c17e56a7
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_new_delete.cpp
@@ -0,0 +1,66 @@
+//===-- hwasan_new_delete.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+
+#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+
+#include <stddef.h>
+
+using namespace __hwasan;
+
+// Fake std::nothrow_t to avoid including <new>.
+namespace std {
+ struct nothrow_t {};
+} // namespace std
+
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_malloc(size, &stack);\
+ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
+ return res
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY \
+ GET_MALLOC_STACK_TRACE; \
+ if (ptr) hwasan_free(ptr, &stack)
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const&) {
+ OPERATOR_DELETE_BODY;
+}
+
+#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
diff --git a/libsanitizer/hwasan/hwasan_poisoning.cpp b/libsanitizer/hwasan/hwasan_poisoning.cpp
new file mode 100644
index 00000000000..2a0816428e7
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_poisoning.cpp
@@ -0,0 +1,52 @@
+//===-- hwasan_poisoning.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_poisoning.h"
+
+#include "hwasan_mapping.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_linux.h"
+
+namespace __hwasan {
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ uptr shadow_start = MemToShadow(p);
+ uptr shadow_size = MemToShadowSize(size);
+
+ uptr page_size = GetPageSizeCached();
+ uptr page_start = RoundUpTo(shadow_start, page_size);
+ uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+ uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+ if (SANITIZER_LINUX &&
+ UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+ internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+ internal_memset((void *)page_end, tag,
+ shadow_start + shadow_size - page_end);
+ // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+ // Linux.
+ ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+ } else {
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ }
+ return AddTagToPointer(p, tag);
+}
+
+uptr TagMemory(uptr p, uptr size, tag_t tag) {
+ uptr start = RoundDownTo(p, kShadowAlignment);
+ uptr end = RoundUpTo(p + size, kShadowAlignment);
+ return TagMemoryAligned(start, end - start, tag);
+}
+
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_poisoning.h b/libsanitizer/hwasan/hwasan_poisoning.h
new file mode 100644
index 00000000000..61751f7d252
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_poisoning.h
@@ -0,0 +1,24 @@
+//===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_POISONING_H
+#define HWASAN_POISONING_H
+
+#include "hwasan.h"
+
+namespace __hwasan {
+uptr TagMemory(uptr p, uptr size, tag_t tag);
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag);
+
+} // namespace __hwasan
+
+#endif // HWASAN_POISONING_H
diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp
new file mode 100644
index 00000000000..606139f2e17
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_report.cpp
@@ -0,0 +1,572 @@
+//===-- hwasan_report.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Error reporting.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stacktrace_printer.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+class ScopedReport {
+ public:
+ ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) {
+ BlockingMutexLock lock(&error_message_lock_);
+ error_message_ptr_ = fatal ? &error_message_ : nullptr;
+ ++hwasan_report_count;
+ }
+
+ ~ScopedReport() {
+ {
+ BlockingMutexLock lock(&error_message_lock_);
+ if (fatal)
+ SetAbortMessage(error_message_.data());
+ error_message_ptr_ = nullptr;
+ }
+ if (common_flags()->print_module_map >= 2 ||
+ (fatal && common_flags()->print_module_map))
+ DumpProcessMap();
+ if (fatal)
+ Die();
+ }
+
+ static void MaybeAppendToErrorMessage(const char *msg) {
+ BlockingMutexLock lock(&error_message_lock_);
+ if (!error_message_ptr_)
+ return;
+ uptr len = internal_strlen(msg);
+ uptr old_size = error_message_ptr_->size();
+ error_message_ptr_->resize(old_size + len);
+ // overwrite old trailing '\0', keep new trailing '\0' untouched.
+ internal_memcpy(&(*error_message_ptr_)[old_size - 1], msg, len);
+ }
+ private:
+ ScopedErrorReportLock error_report_lock_;
+ InternalMmapVector<char> error_message_;
+ bool fatal;
+
+ static InternalMmapVector<char> *error_message_ptr_;
+ static BlockingMutex error_message_lock_;
+};
+
+InternalMmapVector<char> *ScopedReport::error_message_ptr_;
+BlockingMutex ScopedReport::error_message_lock_;
+
+// If there is an active ScopedReport, append to its error message.
+void AppendToErrorMessageBuffer(const char *buffer) {
+ ScopedReport::MaybeAppendToErrorMessage(buffer);
+}
+
+static StackTrace GetStackTraceFromId(u32 id) {
+ CHECK(id);
+ StackTrace res = StackDepotGet(id);
+ CHECK(res.trace);
+ return res;
+}
+
+// A RAII object that holds a copy of the current thread stack ring buffer.
+// The actual stack buffer may change while we are iterating over it (for
+// example, Printf may call syslog() which can itself be built with hwasan).
+class SavedStackAllocations {
+ public:
+ SavedStackAllocations(StackAllocationsRingBuffer *rb) {
+ uptr size = rb->size() * sizeof(uptr);
+ void *storage =
+ MmapAlignedOrDieOnFatalError(size, size * 2, "saved stack allocations");
+ new (&rb_) StackAllocationsRingBuffer(*rb, storage);
+ }
+
+ ~SavedStackAllocations() {
+ StackAllocationsRingBuffer *rb = get();
+ UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr));
+ }
+
+ StackAllocationsRingBuffer *get() {
+ return (StackAllocationsRingBuffer *)&rb_;
+ }
+
+ private:
+ uptr rb_;
+};
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+ Decorator() : SanitizerCommonDecorator() { }
+ const char *Access() { return Blue(); }
+ const char *Allocation() const { return Magenta(); }
+ const char *Origin() const { return Magenta(); }
+ const char *Name() const { return Green(); }
+ const char *Location() { return Green(); }
+ const char *Thread() { return Green(); }
+};
+
+// Returns the index of the rb element that matches tagged_addr (plus one),
+// or zero if found nothing.
+uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb,
+ uptr tagged_addr,
+ HeapAllocationRecord *har) {
+ if (!rb) return 0;
+ for (uptr i = 0, size = rb->size(); i < size; i++) {
+ auto h = (*rb)[i];
+ if (h.tagged_addr <= tagged_addr &&
+ h.tagged_addr + h.requested_size > tagged_addr) {
+ *har = h;
+ return i + 1;
+ }
+ }
+ return 0;
+}
+
+static void PrintStackAllocations(StackAllocationsRingBuffer *sa,
+ tag_t addr_tag, uptr untagged_addr) {
+ uptr frames = Min((uptr)flags()->stack_history_size, sa->size());
+ bool found_local = false;
+ for (uptr i = 0; i < frames; i++) {
+ const uptr *record_addr = &(*sa)[i];
+ uptr record = *record_addr;
+ if (!record)
+ break;
+ tag_t base_tag =
+ reinterpret_cast<uptr>(record_addr) >> kRecordAddrBaseTagShift;
+ uptr fp = (record >> kRecordFPShift) << kRecordFPLShift;
+ uptr pc_mask = (1ULL << kRecordFPShift) - 1;
+ uptr pc = record & pc_mask;
+ FrameInfo frame;
+ if (Symbolizer::GetOrInit()->SymbolizeFrame(pc, &frame)) {
+ for (LocalInfo &local : frame.locals) {
+ if (!local.has_frame_offset || !local.has_size || !local.has_tag_offset)
+ continue;
+ tag_t obj_tag = base_tag ^ local.tag_offset;
+ if (obj_tag != addr_tag)
+ continue;
+ // Calculate the offset from the object address to the faulting
+ // address. Because we only store bits 4-19 of FP (bits 0-3 are
+ // guaranteed to be zero), the calculation is performed mod 2^20 and may
+ // harmlessly underflow if the address mod 2^20 is below the object
+ // address.
+ uptr obj_offset =
+ (untagged_addr - fp - local.frame_offset) & (kRecordFPModulus - 1);
+ if (obj_offset >= local.size)
+ continue;
+ if (!found_local) {
+ Printf("Potentially referenced stack objects:\n");
+ found_local = true;
+ }
+ Printf(" %s in %s %s:%d\n", local.name, local.function_name,
+ local.decl_file, local.decl_line);
+ }
+ frame.Clear();
+ }
+ }
+
+ if (found_local)
+ return;
+
+ // We didn't find any locals. Most likely we don't have symbols, so dump
+ // the information that we have for offline analysis.
+ InternalScopedString frame_desc(GetPageSizeCached() * 2);
+ Printf("Previously allocated frames:\n");
+ for (uptr i = 0; i < frames; i++) {
+ const uptr *record_addr = &(*sa)[i];
+ uptr record = *record_addr;
+ if (!record)
+ break;
+ uptr pc_mask = (1ULL << 48) - 1;
+ uptr pc = record & pc_mask;
+ frame_desc.append(" record_addr:0x%zx record:0x%zx",
+ reinterpret_cast<uptr>(record_addr), record);
+ if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) {
+ RenderFrame(&frame_desc, " %F %L\n", 0, frame->info,
+ common_flags()->symbolize_vs_style,
+ common_flags()->strip_path_prefix);
+ frame->ClearAll();
+ }
+ Printf("%s", frame_desc.data());
+ frame_desc.clear();
+ }
+}
+
+// Returns true if tag == *tag_ptr, reading tags from short granules if
+// necessary. This may return a false positive if tags 1-15 are used as a
+// regular tag rather than a short granule marker.
+static bool TagsEqual(tag_t tag, tag_t *tag_ptr) {
+ if (tag == *tag_ptr)
+ return true;
+ if (*tag_ptr == 0 || *tag_ptr > kShadowAlignment - 1)
+ return false;
+ uptr mem = ShadowToMem(reinterpret_cast<uptr>(tag_ptr));
+ tag_t inline_tag = *reinterpret_cast<tag_t *>(mem + kShadowAlignment - 1);
+ return tag == inline_tag;
+}
+
+void PrintAddressDescription(
+ uptr tagged_addr, uptr access_size,
+ StackAllocationsRingBuffer *current_stack_allocations) {
+ Decorator d;
+ int num_descriptions_printed = 0;
+ uptr untagged_addr = UntagAddr(tagged_addr);
+
+ // Print some very basic information about the address, if it's a heap.
+ HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
+ if (uptr beg = chunk.Beg()) {
+ uptr size = chunk.ActualSize();
+ Printf("%s[%p,%p) is a %s %s heap chunk; "
+ "size: %zd offset: %zd\n%s",
+ d.Location(),
+ beg, beg + size,
+ chunk.FromSmallHeap() ? "small" : "large",
+ chunk.IsAllocated() ? "allocated" : "unallocated",
+ size, untagged_addr - beg,
+ d.Default());
+ }
+
+ // Check if this looks like a heap buffer overflow by scanning
+ // the shadow left and right and looking for the first adjacent
+ // object with a different memory tag. If that tag matches addr_tag,
+ // check the allocator if it has a live chunk there.
+ tag_t addr_tag = GetTagFromPointer(tagged_addr);
+ tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
+ tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
+ for (int i = 0; i < 1000; i++) {
+ if (TagsEqual(addr_tag, left)) {
+ candidate = left;
+ break;
+ }
+ --left;
+ if (TagsEqual(addr_tag, right)) {
+ candidate = right;
+ break;
+ }
+ ++right;
+ }
+
+ if (candidate) {
+ uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
+ HwasanChunkView chunk = FindHeapChunkByAddress(mem);
+ if (chunk.IsAllocated()) {
+ Printf("%s", d.Location());
+ Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n",
+ untagged_addr,
+ candidate == left ? untagged_addr - chunk.End()
+ : chunk.Beg() - untagged_addr,
+ candidate == left ? "right" : "left", chunk.UsedSize(),
+ chunk.Beg(), chunk.End());
+ Printf("%s", d.Allocation());
+ Printf("allocated here:\n");
+ Printf("%s", d.Default());
+ GetStackTraceFromId(chunk.GetAllocStackId()).Print();
+ num_descriptions_printed++;
+ } else {
+ // Check whether the address points into a loaded library. If so, this is
+ // most likely a global variable.
+ const char *module_name;
+ uptr module_address;
+ Symbolizer *sym = Symbolizer::GetOrInit();
+ if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
+ &module_address)) {
+ DataInfo info;
+ if (sym->SymbolizeData(mem, &info) && info.start) {
+ Printf(
+ "%p is located %zd bytes to the %s of %zd-byte global variable "
+ "%s [%p,%p) in %s\n",
+ untagged_addr,
+ candidate == left ? untagged_addr - (info.start + info.size)
+ : info.start - untagged_addr,
+ candidate == left ? "right" : "left", info.size, info.name,
+ info.start, info.start + info.size, module_name);
+ } else {
+ Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+ untagged_addr, candidate == left ? "right" : "left",
+ module_name, module_address);
+ }
+ num_descriptions_printed++;
+ }
+ }
+ }
+
+ hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
+ // Scan all threads' ring buffers to find if it's a heap-use-after-free.
+ HeapAllocationRecord har;
+ if (uptr D = FindHeapAllocation(t->heap_allocations(), tagged_addr, &har)) {
+ Printf("%s", d.Location());
+ Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
+ untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
+ har.requested_size, UntagAddr(har.tagged_addr),
+ UntagAddr(har.tagged_addr) + har.requested_size);
+ Printf("%s", d.Allocation());
+ Printf("freed by thread T%zd here:\n", t->unique_id());
+ Printf("%s", d.Default());
+ GetStackTraceFromId(har.free_context_id).Print();
+
+ Printf("%s", d.Allocation());
+ Printf("previously allocated here:\n", t);
+ Printf("%s", d.Default());
+ GetStackTraceFromId(har.alloc_context_id).Print();
+
+ // Print a developer note: the index of this heap object
+ // in the thread's deallocation ring buffer.
+ Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", D,
+ flags()->heap_history_size);
+
+ t->Announce();
+ num_descriptions_printed++;
+ }
+
+ // Very basic check for stack memory.
+ if (t->AddrIsInStack(untagged_addr)) {
+ Printf("%s", d.Location());
+ Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
+ t->unique_id());
+ Printf("%s", d.Default());
+ t->Announce();
+
+ auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+ ? current_stack_allocations
+ : t->stack_allocations();
+ PrintStackAllocations(sa, addr_tag, untagged_addr);
+ num_descriptions_printed++;
+ }
+ });
+
+ // Print the remaining threads, as an extra information, 1 line per thread.
+ hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); });
+
+ if (!num_descriptions_printed)
+ // We exhausted our possibilities. Bail out.
+ Printf("HWAddressSanitizer can not describe address in more detail.\n");
+}
+
+void ReportStats() {}
+
+static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows,
+ void (*print_tag)(InternalScopedString &s,
+ tag_t *tag)) {
+ const uptr row_len = 16; // better be power of two.
+ tag_t *center_row_beg = reinterpret_cast<tag_t *>(
+ RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len));
+ tag_t *beg_row = center_row_beg - row_len * (num_rows / 2);
+ tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2);
+ InternalScopedString s(GetPageSizeCached() * 8);
+ for (tag_t *row = beg_row; row < end_row; row += row_len) {
+ s.append("%s", row == center_row_beg ? "=>" : " ");
+ for (uptr i = 0; i < row_len; i++) {
+ s.append("%s", row + i == tag_ptr ? "[" : " ");
+ print_tag(s, &row[i]);
+ s.append("%s", row + i == tag_ptr ? "]" : " ");
+ }
+ s.append("%s\n", row == center_row_beg ? "<=" : " ");
+ }
+ Printf("%s", s.data());
+}
+
+static void PrintTagsAroundAddr(tag_t *tag_ptr) {
+ Printf(
+ "Memory tags around the buggy address (one tag corresponds to %zd "
+ "bytes):\n", kShadowAlignment);
+ PrintTagInfoAroundAddr(tag_ptr, 17, [](InternalScopedString &s, tag_t *tag) {
+ s.append("%02x", *tag);
+ });
+
+ Printf(
+ "Tags for short granules around the buggy address (one tag corresponds "
+ "to %zd bytes):\n",
+ kShadowAlignment);
+ PrintTagInfoAroundAddr(tag_ptr, 3, [](InternalScopedString &s, tag_t *tag) {
+ if (*tag >= 1 && *tag <= kShadowAlignment) {
+ uptr granule_addr = ShadowToMem(reinterpret_cast<uptr>(tag));
+ s.append("%02x",
+ *reinterpret_cast<u8 *>(granule_addr + kShadowAlignment - 1));
+ } else {
+ s.append("..");
+ }
+ });
+ Printf(
+ "See "
+ "https://clang.llvm.org/docs/";
+ "HardwareAssistedAddressSanitizerDesign.html#short-granules for a "
+ "description of short granule tags\n");
+}
+
+void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) {
+ ScopedReport R(flags()->halt_on_error);
+
+ uptr untagged_addr = UntagAddr(tagged_addr);
+ tag_t ptr_tag = GetTagFromPointer(tagged_addr);
+ tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
+ tag_t mem_tag = *tag_ptr;
+ Decorator d;
+ Printf("%s", d.Error());
+ uptr pc = stack->size ? stack->trace[0] : 0;
+ const char *bug_type = "invalid-free";
+ Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type,
+ untagged_addr, pc);
+ Printf("%s", d.Access());
+ Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag);
+ Printf("%s", d.Default());
+
+ stack->Print();
+
+ PrintAddressDescription(tagged_addr, 0, nullptr);
+
+ PrintTagsAroundAddr(tag_ptr);
+
+ ReportErrorSummary(bug_type, stack);
+}
+
+void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
+ const u8 *expected) {
+ uptr tail_size = kShadowAlignment - (orig_size % kShadowAlignment);
+ ScopedReport R(flags()->halt_on_error);
+ Decorator d;
+ uptr untagged_addr = UntagAddr(tagged_addr);
+ Printf("%s", d.Error());
+ const char *bug_type = "allocation-tail-overwritten";
+ Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName,
+ bug_type, untagged_addr, untagged_addr + orig_size, orig_size);
+ Printf("\n%s", d.Default());
+ stack->Print();
+ HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
+ if (chunk.Beg()) {
+ Printf("%s", d.Allocation());
+ Printf("allocated here:\n");
+ Printf("%s", d.Default());
+ GetStackTraceFromId(chunk.GetAllocStackId()).Print();
+ }
+
+ InternalScopedString s(GetPageSizeCached() * 8);
+ CHECK_GT(tail_size, 0U);
+ CHECK_LT(tail_size, kShadowAlignment);
+ u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size);
+ s.append("Tail contains: ");
+ for (uptr i = 0; i < kShadowAlignment - tail_size; i++)
+ s.append(".. ");
+ for (uptr i = 0; i < tail_size; i++)
+ s.append("%02x ", tail[i]);
+ s.append("\n");
+ s.append("Expected: ");
+ for (uptr i = 0; i < kShadowAlignment - tail_size; i++)
+ s.append(".. ");
+ for (uptr i = 0; i < tail_size; i++)
+ s.append("%02x ", expected[i]);
+ s.append("\n");
+ s.append(" ");
+ for (uptr i = 0; i < kShadowAlignment - tail_size; i++)
+ s.append(" ");
+ for (uptr i = 0; i < tail_size; i++)
+ s.append("%s ", expected[i] != tail[i] ? "^^" : " ");
+
+ s.append("\nThis error occurs when a buffer overflow overwrites memory\n"
+ "to the right of a heap object, but within the %zd-byte granule, e.g.\n"
+ " char *x = new char[20];\n"
+ " x[25] = 42;\n"
+ "%s does not detect such bugs in uninstrumented code at the time of write,"
+ "\nbut can detect them at the time of free/delete.\n"
+ "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0\n",
+ kShadowAlignment, SanitizerToolName);
+ Printf("%s", s.data());
+ GetCurrentThread()->Announce();
+
+ tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
+ PrintTagsAroundAddr(tag_ptr);
+
+ ReportErrorSummary(bug_type, stack);
+}
+
+void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
+ bool is_store, bool fatal, uptr *registers_frame) {
+ ScopedReport R(fatal);
+ SavedStackAllocations current_stack_allocations(
+ GetCurrentThread()->stack_allocations());
+
+ Decorator d;
+ Printf("%s", d.Error());
+ uptr untagged_addr = UntagAddr(tagged_addr);
+ // TODO: when possible, try to print heap-use-after-free, etc.
+ const char *bug_type = "tag-mismatch";
+ uptr pc = stack->size ? stack->trace[0] : 0;
+ Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type,
+ untagged_addr, pc);
+
+ Thread *t = GetCurrentThread();
+
+ sptr offset =
+ __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size);
+ CHECK(offset >= 0 && offset < static_cast<sptr>(access_size));
+ tag_t ptr_tag = GetTagFromPointer(tagged_addr);
+ tag_t *tag_ptr =
+ reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset));
+ tag_t mem_tag = *tag_ptr;
+
+ Printf("%s", d.Access());
+ Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
+ is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
+ mem_tag, t->unique_id());
+ if (offset != 0)
+ Printf("Invalid access starting at offset [%zu, %zu)\n", offset,
+ Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale)));
+ Printf("%s", d.Default());
+
+ stack->Print();
+
+ PrintAddressDescription(tagged_addr, access_size,
+ current_stack_allocations.get());
+ t->Announce();
+
+ PrintTagsAroundAddr(tag_ptr);
+
+ if (registers_frame)
+ ReportRegisters(registers_frame, pc);
+
+ ReportErrorSummary(bug_type, stack);
+}
+
+// See the frame breakdown defined in __hwasan_tag_mismatch (from
+// hwasan_tag_mismatch_aarch64.S).
+void ReportRegisters(uptr *frame, uptr pc) {
+ Printf("Registers where the failure occurred (pc %p):\n", pc);
+
+ // We explicitly print a single line (4 registers/line) each iteration to
+ // reduce the amount of logcat error messages printed. Each Printf() will
+ // result in a new logcat line, irrespective of whether a newline is present,
+ // and so we wish to reduce the number of Printf() calls we have to make.
+ Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n",
+ frame[0], frame[1], frame[2], frame[3]);
+ Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n",
+ frame[4], frame[5], frame[6], frame[7]);
+ Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n",
+ frame[8], frame[9], frame[10], frame[11]);
+ Printf(" x12 %016llx x13 %016llx x14 %016llx x15 %016llx\n",
+ frame[12], frame[13], frame[14], frame[15]);
+ Printf(" x16 %016llx x17 %016llx x18 %016llx x19 %016llx\n",
+ frame[16], frame[17], frame[18], frame[19]);
+ Printf(" x20 %016llx x21 %016llx x22 %016llx x23 %016llx\n",
+ frame[20], frame[21], frame[22], frame[23]);
+ Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n",
+ frame[24], frame[25], frame[26], frame[27]);
+ Printf(" x28 %016llx x29 %016llx x30 %016llx\n",
+ frame[28], frame[29], frame[30]);
+}
+
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_report.h b/libsanitizer/hwasan/hwasan_report.h
new file mode 100644
index 00000000000..de86c38fc01
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_report.h
@@ -0,0 +1,35 @@
+//===-- hwasan_report.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. HWASan-private header for error
+/// reporting functions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_REPORT_H
+#define HWASAN_REPORT_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+namespace __hwasan {
+
+void ReportStats();
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store, bool fatal, uptr *registers_frame);
+void ReportInvalidFree(StackTrace *stack, uptr addr);
+void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size,
+ const u8 *expected);
+void ReportRegisters(uptr *registers_frame, uptr pc);
+void ReportAtExitStatistics();
+
+
+} // namespace __hwasan
+
+#endif // HWASAN_REPORT_H
diff --git a/libsanitizer/hwasan/hwasan_setjmp.S b/libsanitizer/hwasan/hwasan_setjmp.S
new file mode 100644
index 00000000000..0c135433194
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_setjmp.S
@@ -0,0 +1,100 @@
+//===-- hwasan_setjmp.S --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+#include "sanitizer_common/sanitizer_platform.h"
+
+// We want to save the context of the calling function.
+// That requires
+// 1) No modification of the link register by this function.
+// 2) No modification of the stack pointer by this function.
+// 3) (no modification of any other saved register, but that's not really going
+// to occur, and hence isn't as much of a worry).
+//
+// There's essentially no way to ensure that the compiler will not modify the
+// stack pointer when compiling a C function.
+// Hence we have to write this function in assembly.
+
+.section .text
+.file "hwasan_setjmp.S"
+
+.global __interceptor_setjmp
+ASM_TYPE_FUNCTION(__interceptor_setjmp)
+__interceptor_setjmp:
+ CFI_STARTPROC
+ mov x1, #0
+ b __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp)
+
+#if SANITIZER_ANDROID
+// Bionic also defines a function `setjmp` that calls `sigsetjmp` saving the
+// current signal.
+.global __interceptor_setjmp_bionic
+ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic)
+__interceptor_setjmp_bionic:
+ CFI_STARTPROC
+ mov x1, #1
+ b __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp_bionic)
+#endif
+
+.global __interceptor_sigsetjmp
+ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
+__interceptor_sigsetjmp:
+ CFI_STARTPROC
+ stp x19, x20, [x0, #0<<3]
+ stp x21, x22, [x0, #2<<3]
+ stp x23, x24, [x0, #4<<3]
+ stp x25, x26, [x0, #6<<3]
+ stp x27, x28, [x0, #8<<3]
+ stp x29, x30, [x0, #10<<3]
+ stp d8, d9, [x0, #14<<3]
+ stp d10, d11, [x0, #16<<3]
+ stp d12, d13, [x0, #18<<3]
+ stp d14, d15, [x0, #20<<3]
+ mov x2, sp
+ str x2, [x0, #13<<3]
+ // We always have the second argument to __sigjmp_save (savemask) set, since
+ // the _setjmp function above has set it for us as `false`.
+ // This function is defined in hwasan_interceptors.cc
+ b __sigjmp_save
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_sigsetjmp)
+
+
+.macro ALIAS first second
+ .globl \second
+ .equ \second\(), \first
+.endm
+
+#if SANITIZER_ANDROID
+ALIAS __interceptor_sigsetjmp, sigsetjmp
+.weak sigsetjmp
+
+ALIAS __interceptor_setjmp_bionic, setjmp
+.weak setjmp
+#else
+ALIAS __interceptor_sigsetjmp, __sigsetjmp
+.weak __sigsetjmp
+#endif
+
+ALIAS __interceptor_setjmp, _setjmp
+.weak _setjmp
+#endif
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
diff --git a/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S b/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S
new file mode 100644
index 00000000000..08df12736bb
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S
@@ -0,0 +1,152 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
+// The responsibility of the HWASan entry point in compiler-rt is to primarily
+// readjust the stack from the callee and save the current register values to
+// the stack.
+// This entry point function should be called from a __hwasan_check_* symbol.
+// These are generated during a lowering pass in the backend, and are found in
+// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for
+// further information.
+// The __hwasan_check_* caller of this function should have expanded the stack
+// and saved the previous values of x0, x1, x29, and x30. This function will
+// "consume" these saved values and treats it as part of its own stack frame.
+// In this sense, the __hwasan_check_* callee and this function "share" a stack
+// frame. This allows us to omit having unwinding information (.cfi_*) present
+// in every __hwasan_check_* function, therefore reducing binary size. This is
+// particularly important as hwasan_check_* instances are duplicated in every
+// translation unit where HWASan is enabled.
+// This function calls HwasanTagMismatch to step back into the C++ code that
+// completes the stack unwinding and error printing. This function is is not
+// permitted to return.
+
+
+// Frame from __hwasan_check_:
+// | ... |
+// | ... |
+// | Previous stack frames... |
+// +=================================+
+// | Unused 8-bytes for maintaining |
+// | 16-byte SP alignment. |
+// +---------------------------------+
+// | Return address (x30) for caller |
+// | of __hwasan_check_*. |
+// +---------------------------------+
+// | Frame address (x29) for caller |
+// | of __hwasan_check_* |
+// +---------------------------------+ <-- [SP + 232]
+// | ... |
+// | |
+// | Stack frame space for x2 - x28. |
+// | |
+// | ... |
+// +---------------------------------+ <-- [SP + 16]
+// | |
+// | Saved x1, as __hwasan_check_* |
+// | clobbers it. |
+// +---------------------------------+
+// | Saved x0, likewise above. |
+// +---------------------------------+ <-- [x30 / SP]
+
+// This function takes two arguments:
+// * x0: The data address.
+// * x1: The encoded access info for the failing access.
+
+// This function has two entry points. The first, __hwasan_tag_mismatch, is used
+// by clients that were compiled without short tag checks (i.e. binaries built
+// by older compilers and binaries targeting older runtimes). In this case the
+// outlined tag check will be missing the code handling short tags (which won't
+// be used in the binary's own stack variables but may be used on the heap
+// or stack variables in other binaries), so the check needs to be done here.
+//
+// The second, __hwasan_tag_mismatch_v2, is used by binaries targeting newer
+// runtimes. This entry point bypasses the short tag check since it will have
+// already been done as part of the outlined tag check. Since tag mismatches are
+// uncommon, there isn't a significant performance benefit to being able to
+// bypass the check; the main benefits are that we can sometimes avoid
+// clobbering the x17 register in error reports, and that the program will have
+// a runtime dependency on the __hwasan_tag_mismatch_v2 symbol therefore it will
+// fail to start up given an older (i.e. incompatible) runtime.
+.section .text
+.file "hwasan_tag_mismatch_aarch64.S"
+.global __hwasan_tag_mismatch
+.type __hwasan_tag_mismatch, %function
+__hwasan_tag_mismatch:
+ // Compute the granule position one past the end of the access.
+ mov x16, #1
+ and x17, x1, #0xf
+ lsl x16, x16, x17
+ and x17, x0, #0xf
+ add x17, x16, x17
+
+ // Load the shadow byte again and check whether it is a short tag within the
+ // range of the granule position computed above.
+ ubfx x16, x0, #4, #52
+ ldrb w16, [x9, x16]
+ cmp w16, #0xf
+ b.hi __hwasan_tag_mismatch_v2
+ cmp w16, w17
+ b.lo __hwasan_tag_mismatch_v2
+
+ // Load the real tag from the last byte of the granule and compare against
+ // the pointer tag.
+ orr x16, x0, #0xf
+ ldrb w16, [x16]
+ cmp x16, x0, lsr #56
+ b.ne __hwasan_tag_mismatch_v2
+
+ // Restore x0, x1 and sp to their values from before the __hwasan_tag_mismatch
+ // call and resume execution.
+ ldp x0, x1, [sp], #256
+ ret
+
+.global __hwasan_tag_mismatch_v2
+.type __hwasan_tag_mismatch_v2, %function
+__hwasan_tag_mismatch_v2:
+ CFI_STARTPROC
+
+ // Set the CFA to be the return address for caller of __hwasan_check_*. Note
+ // that we do not emit CFI predicates to describe the contents of this stack
+ // frame, as this proxy entry point should never be debugged. The contents
+ // are static and are handled by the unwinder after calling
+ // __hwasan_tag_mismatch. The frame pointer is already correctly setup
+ // by __hwasan_check_*.
+ add x29, sp, #232
+ CFI_DEF_CFA(w29, 24)
+ CFI_OFFSET(w30, -16)
+ CFI_OFFSET(w29, -24)
+
+ // Save the rest of the registers into the preallocated space left by
+ // __hwasan_check.
+ str x28, [sp, #224]
+ stp x26, x27, [sp, #208]
+ stp x24, x25, [sp, #192]
+ stp x22, x23, [sp, #176]
+ stp x20, x21, [sp, #160]
+ stp x18, x19, [sp, #144]
+ stp x16, x17, [sp, #128]
+ stp x14, x15, [sp, #112]
+ stp x12, x13, [sp, #96]
+ stp x10, x11, [sp, #80]
+ stp x8, x9, [sp, #64]
+ stp x6, x7, [sp, #48]
+ stp x4, x5, [sp, #32]
+ stp x2, x3, [sp, #16]
+
+ // Pass the address of the frame to __hwasan_tag_mismatch4, so that it can
+ // extract the saved registers from this frame without having to worry about
+ // finding this frame.
+ mov x2, sp
+
+ bl __hwasan_tag_mismatch4
+ CFI_ENDPROC
+
+.Lfunc_end0:
+ .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch
+
+#endif // defined(__aarch64__)
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
diff --git a/libsanitizer/hwasan/hwasan_thread.cpp b/libsanitizer/hwasan/hwasan_thread.cpp
new file mode 100644
index 00000000000..cabf614c005
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_thread.cpp
@@ -0,0 +1,127 @@
+
+#include "hwasan.h"
+#include "hwasan_mapping.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_interface_internal.h"
+
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+
+namespace __hwasan {
+
+static u32 RandomSeed() {
+ u32 seed;
+ do {
+ if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(&seed), sizeof(seed),
+ /*blocking=*/false))) {
+ seed = static_cast<u32>(
+ (NanoTime() >> 12) ^
+ (reinterpret_cast<uptr>(__builtin_frame_address(0)) >> 4));
+ }
+ } while (!seed);
+ return seed;
+}
+
+void Thread::InitRandomState() {
+ random_state_ = flags()->random_tags ? RandomSeed() : unique_id_;
+
+ // Push a random number of zeros onto the ring buffer so that the first stack
+ // tag base will be random.
+ for (tag_t i = 0, e = GenerateRandomTag(); i != e; ++i)
+ stack_allocations_->push(0);
+}
+
+void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
+ static u64 unique_id;
+ unique_id_ = unique_id++;
+ if (auto sz = flags()->heap_history_size)
+ heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
+
+ HwasanTSDThreadInit(); // Only needed with interceptors.
+ uptr *ThreadLong = GetCurrentThreadLongPtr();
+ // The following implicitly sets (this) as the current thread.
+ stack_allocations_ = new (ThreadLong)
+ StackAllocationsRingBuffer((void *)stack_buffer_start, stack_buffer_size);
+ // Check that it worked.
+ CHECK_EQ(GetCurrentThread(), this);
+
+ // ScopedTaggingDisable needs GetCurrentThread to be set up.
+ ScopedTaggingDisabler disabler;
+
+ uptr tls_size;
+ uptr stack_size;
+ GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
+ &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+
+ if (stack_bottom_) {
+ int local;
+ CHECK(AddrIsInStack((uptr)&local));
+ CHECK(MemIsApp(stack_bottom_));
+ CHECK(MemIsApp(stack_top_ - 1));
+ }
+
+ if (flags()->verbose_threads) {
+ if (IsMainThread()) {
+ Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n",
+ sizeof(Thread), heap_allocations_->SizeInBytes(),
+ stack_allocations_->size() * sizeof(uptr));
+ }
+ Print("Creating : ");
+ }
+}
+
+void Thread::ClearShadowForThreadStackAndTLS() {
+ if (stack_top_ != stack_bottom_)
+ TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0);
+ if (tls_begin_ != tls_end_)
+ TagMemory(tls_begin_, tls_end_ - tls_begin_, 0);
+}
+
+void Thread::Destroy() {
+ if (flags()->verbose_threads)
+ Print("Destroying: ");
+ AllocatorSwallowThreadLocalCache(allocator_cache());
+ ClearShadowForThreadStackAndTLS();
+ if (heap_allocations_)
+ heap_allocations_->Delete();
+ DTLS_Destroy();
+}
+
+void Thread::Print(const char *Prefix) {
+ Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix,
+ unique_id_, this, stack_bottom(), stack_top(),
+ stack_top() - stack_bottom(),
+ tls_begin(), tls_end());
+}
+
+static u32 xorshift(u32 state) {
+ state ^= state << 13;
+ state ^= state >> 17;
+ state ^= state << 5;
+ return state;
+}
+
+// Generate a (pseudo-)random non-zero tag.
+tag_t Thread::GenerateRandomTag() {
+ if (tagging_disabled_) return 0;
+ tag_t tag;
+ do {
+ if (flags()->random_tags) {
+ if (!random_buffer_)
+ random_buffer_ = random_state_ = xorshift(random_state_);
+ CHECK(random_buffer_);
+ tag = random_buffer_ & 0xFF;
+ random_buffer_ >>= 8;
+ } else {
+ tag = random_state_ = (random_state_ + 1) & 0xFF;
+ }
+ } while (!tag);
+ return tag;
+}
+
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_thread.h b/libsanitizer/hwasan/hwasan_thread.h
new file mode 100644
index 00000000000..42c1e9e124b
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_thread.h
@@ -0,0 +1,112 @@
+//===-- hwasan_thread.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_THREAD_H
+#define HWASAN_THREAD_H
+
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_ring_buffer.h"
+
+namespace __hwasan {
+
+typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer;
+
+class Thread {
+ public:
+ void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself.
+ void InitRandomState();
+ void Destroy();
+
+ uptr stack_top() { return stack_top_; }
+ uptr stack_bottom() { return stack_bottom_; }
+ uptr stack_size() { return stack_top() - stack_bottom(); }
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ bool IsMainThread() { return unique_id_ == 0; }
+
+ bool AddrIsInStack(uptr addr) {
+ return addr >= stack_bottom_ && addr < stack_top_;
+ }
+
+ bool InSignalHandler() { return in_signal_handler_; }
+ void EnterSignalHandler() { in_signal_handler_++; }
+ void LeaveSignalHandler() { in_signal_handler_--; }
+
+ bool InSymbolizer() { return in_symbolizer_; }
+ void EnterSymbolizer() { in_symbolizer_++; }
+ void LeaveSymbolizer() { in_symbolizer_--; }
+
+ AllocatorCache *allocator_cache() { return &allocator_cache_; }
+ HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; }
+ StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; }
+
+ tag_t GenerateRandomTag();
+
+ void DisableTagging() { tagging_disabled_++; }
+ void EnableTagging() { tagging_disabled_--; }
+ bool TaggingIsDisabled() const { return tagging_disabled_; }
+
+ u64 unique_id() const { return unique_id_; }
+ void Announce() {
+ if (announced_) return;
+ announced_ = true;
+ Print("Thread: ");
+ }
+
+ uptr &vfork_spill() { return vfork_spill_; }
+
+ private:
+ // NOTE: There is no Thread constructor. It is allocated
+ // via mmap() and *must* be valid in zero-initialized state.
+ void ClearShadowForThreadStackAndTLS();
+ void Print(const char *prefix);
+ uptr vfork_spill_;
+ uptr stack_top_;
+ uptr stack_bottom_;
+ uptr tls_begin_;
+ uptr tls_end_;
+
+ unsigned in_signal_handler_;
+ unsigned in_symbolizer_;
+
+ u32 random_state_;
+ u32 random_buffer_;
+
+ AllocatorCache allocator_cache_;
+ HeapAllocationsRingBuffer *heap_allocations_;
+ StackAllocationsRingBuffer *stack_allocations_;
+
+ static void InsertIntoThreadList(Thread *t);
+ static void RemoveFromThreadList(Thread *t);
+ Thread *next_; // All live threads form a linked list.
+
+ u64 unique_id_; // counting from zero.
+
+ u32 tagging_disabled_; // if non-zero, malloc uses zero tag in this thread.
+
+ bool announced_;
+
+ friend struct ThreadListHead;
+};
+
+Thread *GetCurrentThread();
+uptr *GetCurrentThreadLongPtr();
+
+struct ScopedTaggingDisabler {
+ ScopedTaggingDisabler() { GetCurrentThread()->DisableTagging(); }
+ ~ScopedTaggingDisabler() { GetCurrentThread()->EnableTagging(); }
+};
+
+} // namespace __hwasan
+
+#endif // HWASAN_THREAD_H
diff --git a/libsanitizer/hwasan/hwasan_thread_list.cpp b/libsanitizer/hwasan/hwasan_thread_list.cpp
new file mode 100644
index 00000000000..a31eee84ed9
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_thread_list.cpp
@@ -0,0 +1,15 @@
+#include "hwasan_thread_list.h"
+
+namespace __hwasan {
+static ALIGNED(16) char thread_list_placeholder[sizeof(HwasanThreadList)];
+static HwasanThreadList *hwasan_thread_list;
+
+HwasanThreadList &hwasanThreadList() { return *hwasan_thread_list; }
+
+void InitThreadList(uptr storage, uptr size) {
+ CHECK(hwasan_thread_list == nullptr);
+ hwasan_thread_list =
+ new (thread_list_placeholder) HwasanThreadList(storage, size);
+}
+
+} // namespace
diff --git a/libsanitizer/hwasan/hwasan_thread_list.h b/libsanitizer/hwasan/hwasan_thread_list.h
new file mode 100644
index 00000000000..914b632d977
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_thread_list.h
@@ -0,0 +1,215 @@
+//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+// HwasanThreadList is a registry for live threads, as well as an allocator for
+// HwasanThread objects and their stack history ring buffers. There are
+// constraints on memory layout of the shadow region and CompactRingBuffer that
+// are part of the ABI contract between compiler-rt and llvm.
+//
+// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment.
+// * All stack ring buffers are located within (2**kShadowBaseAlignment)
+// sized region below and adjacent to the shadow region.
+// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 8), and is
+// aligned to twice its size. The value of N can be different for each buffer.
+//
+// These constrains guarantee that, given an address A of any element of the
+// ring buffer,
+// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1)
+// is the address of the next element of that ring buffer (with wrap-around).
+// And, with K = kShadowBaseAlignment,
+// S = (A | ((1 << K) - 1)) + 1
+// (align up to kShadowBaseAlignment) is the start of the shadow region.
+//
+// These calculations are used in compiler instrumentation to update the ring
+// buffer and obtain the base address of shadow using only two inputs: address
+// of the current element of the ring buffer, and N (i.e. size of the ring
+// buffer). Since the value of N is very limited, we pack both inputs into a
+// single thread-local word as
+// (1 << (N + 56)) | A
+// See the implementation of class CompactRingBuffer, which is what is stored in
+// said thread-local word.
+//
+// Note the unusual way of aligning up the address of the shadow:
+// (A | ((1 << K) - 1)) + 1
+// It is only correct if A is not already equal to the shadow base address, but
+// it saves 2 instructions on AArch64.
+
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_flags.h"
+#include "hwasan_thread.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+
+namespace __hwasan {
+
+static uptr RingBufferSize() {
+ uptr desired_bytes = flags()->stack_history_size * sizeof(uptr);
+ // FIXME: increase the limit to 8 once this bug is fixed:
+ // https://bugs.llvm.org/show_bug.cgi?id=39030
+ for (int shift = 1; shift < 7; ++shift) {
+ uptr size = 4096 * (1ULL << shift);
+ if (size >= desired_bytes)
+ return size;
+ }
+ Printf("stack history size too large: %d\n", flags()->stack_history_size);
+ CHECK(0);
+ return 0;
+}
+
+struct ThreadListHead {
+ Thread *list_;
+
+ ThreadListHead() : list_(nullptr) {}
+
+ void Push(Thread *t) {
+ t->next_ = list_;
+ list_ = t;
+ }
+
+ Thread *Pop() {
+ Thread *t = list_;
+ if (t)
+ list_ = t->next_;
+ return t;
+ }
+
+ void Remove(Thread *t) {
+ Thread **cur = &list_;
+ while (*cur != t) cur = &(*cur)->next_;
+ CHECK(*cur && "thread not found");
+ *cur = (*cur)->next_;
+ }
+
+ template <class CB>
+ void ForEach(CB cb) {
+ Thread *t = list_;
+ while (t) {
+ cb(t);
+ t = t->next_;
+ }
+ }
+};
+
+struct ThreadStats {
+ uptr n_live_threads;
+ uptr total_stack_size;
+};
+
+class HwasanThreadList {
+ public:
+ HwasanThreadList(uptr storage, uptr size)
+ : free_space_(storage), free_space_end_(storage + size) {
+ // [storage, storage + size) is used as a vector of
+ // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.
+ // Each element contains
+ // * a ring buffer at offset 0,
+ // * a Thread object at offset ring_buffer_size_.
+ ring_buffer_size_ = RingBufferSize();
+ thread_alloc_size_ =
+ RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
+ }
+
+ Thread *CreateCurrentThread() {
+ Thread *t;
+ {
+ SpinMutexLock l(&list_mutex_);
+ t = free_list_.Pop();
+ if (t) {
+ uptr start = (uptr)t - ring_buffer_size_;
+ internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
+ } else {
+ t = AllocThread();
+ }
+ live_list_.Push(t);
+ }
+ t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
+ AddThreadStats(t);
+ return t;
+ }
+
+ void DontNeedThread(Thread *t) {
+ uptr start = (uptr)t - ring_buffer_size_;
+ ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);
+ }
+
+ void ReleaseThread(Thread *t) {
+ RemoveThreadStats(t);
+ t->Destroy();
+ SpinMutexLock l(&list_mutex_);
+ live_list_.Remove(t);
+ free_list_.Push(t);
+ DontNeedThread(t);
+ }
+
+ Thread *GetThreadByBufferAddress(uptr p) {
+ return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) +
+ ring_buffer_size_);
+ }
+
+ uptr MemoryUsedPerThread() {
+ uptr res = sizeof(Thread) + ring_buffer_size_;
+ if (auto sz = flags()->heap_history_size)
+ res += HeapAllocationsRingBuffer::SizeInBytes(sz);
+ return res;
+ }
+
+ template <class CB>
+ void VisitAllLiveThreads(CB cb) {
+ SpinMutexLock l(&list_mutex_);
+ live_list_.ForEach(cb);
+ }
+
+ void AddThreadStats(Thread *t) {
+ SpinMutexLock l(&stats_mutex_);
+ stats_.n_live_threads++;
+ stats_.total_stack_size += t->stack_size();
+ }
+
+ void RemoveThreadStats(Thread *t) {
+ SpinMutexLock l(&stats_mutex_);
+ stats_.n_live_threads--;
+ stats_.total_stack_size -= t->stack_size();
+ }
+
+ ThreadStats GetThreadStats() {
+ SpinMutexLock l(&stats_mutex_);
+ return stats_;
+ }
+
+ private:
+ Thread *AllocThread() {
+ uptr align = ring_buffer_size_ * 2;
+ CHECK(IsAligned(free_space_, align));
+ Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
+ free_space_ += thread_alloc_size_;
+ CHECK(free_space_ <= free_space_end_ && "out of thread memory");
+ return t;
+ }
+
+ uptr free_space_;
+ uptr free_space_end_;
+ uptr ring_buffer_size_;
+ uptr thread_alloc_size_;
+
+ ThreadListHead free_list_;
+ ThreadListHead live_list_;
+ SpinMutex list_mutex_;
+
+ ThreadStats stats_;
+ SpinMutex stats_mutex_;
+};
+
+void InitThreadList(uptr storage, uptr size);
+HwasanThreadList &hwasanThreadList();
+
+} // namespace
diff --git a/libsanitizer/hwasan/hwasan_type_test.cpp b/libsanitizer/hwasan/hwasan_type_test.cpp
new file mode 100644
index 00000000000..8cff495bae1
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_type_test.cpp
@@ -0,0 +1,25 @@
+//===-- hwasan_type_test.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Compile-time tests of the internal type definitions.
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "hwasan.h"
+#include <setjmp.h>
+
+#define CHECK_TYPE_SIZE_FITS(TYPE) \
+ COMPILER_CHECK(sizeof(__hw_##TYPE) <= sizeof(TYPE))
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+CHECK_TYPE_SIZE_FITS(jmp_buf);
+CHECK_TYPE_SIZE_FITS(sigjmp_buf);
+#endif
diff --git a/libsanitizer/merge.sh b/libsanitizer/merge.sh
index e583065ca56..9343dfa45b2 100755
--- a/libsanitizer/merge.sh
+++ b/libsanitizer/merge.sh
@@ -73,6 +73,7 @@ merge lib/tsan/rtl tsan
merge lib/sanitizer_common sanitizer_common
merge lib/interception interception
merge lib/ubsan ubsan
+merge lib/hwasan hwasan
# Need to merge lib/builtins/assembly.h file:
mkdir -p builtins
--
2.23.0