[PATCH] Cherry-pick libsanitizer pointer-pair tristate option.

[Martin Liška <mliska at suse dot cz>]

Hi.

This is upstream backport of libsanitizer which makes detect-invalid-pointer-pair
a tristate option.

Patch is preapproved by Jakub.
Patch can bootstrap on ppc64le-redhat-linux and survives regression tests.

Martin

gcc/ChangeLog:

2018-02-05  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Cherry-pick upstream r323995.

gcc/testsuite/ChangeLog:

2018-02-05  Martin Liska  <mliska@suse.cz>

	* c-c++-common/asan/pointer-compare-1.c: Adjust ASAN_OPTIONS
	options.
	* c-c++-common/asan/pointer-compare-2.c: Likewise.
	* c-c++-common/asan/pointer-subtract-1.c: Likewise.
	* c-c++-common/asan/pointer-subtract-2.c: Likewise.
	* c-c++-common/asan/pointer-subtract-3.c: Likewise.
	* c-c++-common/asan/pointer-subtract-4.c: Likewise.
	* c-c++-common/asan/pointer-compare-3.c: New test.

libsanitizer/ChangeLog:

2018-02-05  Martin Liska  <mliska@suse.cz>

	* asan/asan_flags.inc: Cherry-pick upstream r323995.
	* asan/asan_report.cc (CheckForInvalidPointerPair):
	Cherry-pick upstream r323995.
---
 gcc/doc/invoke.texi                                | 10 +++---
 .../c-c++-common/asan/pointer-compare-1.c          |  2 +-
 .../c-c++-common/asan/pointer-compare-2.c          |  2 +-
 .../c-c++-common/asan/pointer-compare-3.c          | 39 ++++++++++++++++++++++
 .../c-c++-common/asan/pointer-subtract-1.c         |  2 +-
 .../c-c++-common/asan/pointer-subtract-2.c         |  2 +-
 .../c-c++-common/asan/pointer-subtract-3.c         |  2 +-
 .../c-c++-common/asan/pointer-subtract-4.c         |  2 +-
 libsanitizer/asan/asan_flags.inc                   |  6 ++--
 libsanitizer/asan/asan_report.cc                   |  6 +++-
 10 files changed, 59 insertions(+), 14 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/asan/pointer-compare-3.c


diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index f3d93367640..cf6d3ae5b99 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -11165,8 +11165,9 @@ The option must be combined with either @option{-fsanitize=kernel-address} or
 The option cannot be combined with @option{-fsanitize=thread}
 and/or @option{-fcheck-pointer-bounds}.
 Note: By default the check is disabled at run time.  To enable it,
-add @code{detect_invalid_pointer_pairs=1} to the environment variable
-@env{ASAN_OPTIONS}.
+add @code{detect_invalid_pointer_pairs=2} to the environment variable
+@env{ASAN_OPTIONS}. Using @code{detect_invalid_pointer_pairs=1} detects
+invalid operation only when both pointers are non-null.
 
 @item -fsanitize=pointer-subtract
 @opindex fsanitize=pointer-subtract
@@ -11176,8 +11177,9 @@ The option must be combined with either @option{-fsanitize=kernel-address} or
 The option cannot be combined with @option{-fsanitize=thread}
 and/or @option{-fcheck-pointer-bounds}.
 Note: By default the check is disabled at run time.  To enable it,
-add @code{detect_invalid_pointer_pairs=1} to the environment variable
-@env{ASAN_OPTIONS}.
+add @code{detect_invalid_pointer_pairs=2} to the environment variable
+@env{ASAN_OPTIONS}. Using @code{detect_invalid_pointer_pairs=1} detects
+invalid operation only when both pointers are non-null.
 
 @item -fsanitize=thread
 @opindex fsanitize=thread
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-compare-1.c b/gcc/testsuite/c-c++-common/asan/pointer-compare-1.c
index 2cc7395241a..cf67fe98bee 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-compare-1.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-compare-1.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1:halt_on_error=0" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2:halt_on_error=0" } */
 /* { dg-options "-fsanitize=address,pointer-compare" } */
 
 volatile int v;
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-compare-2.c b/gcc/testsuite/c-c++-common/asan/pointer-compare-2.c
index 5539087e856..d2142c8f160 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-compare-2.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-compare-2.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1 halt_on_error=1" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2 halt_on_error=1" } */
 /* { dg-options "-fsanitize=address,pointer-compare" } */
 
 volatile int v;
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-compare-3.c b/gcc/testsuite/c-c++-common/asan/pointer-compare-3.c
new file mode 100644
index 00000000000..7156e494aeb
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/asan/pointer-compare-3.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1:halt_on_error=1" } */
+/* { dg-options "-fsanitize=address,pointer-compare" } */
+
+int foo(char *p, char *q) {
+  return p <= q;
+}
+
+char global[8192] = {};
+char small_global[7] = {};
+
+int main() {
+  // Heap allocated memory.
+  char *p = (char *)__builtin_malloc(42);
+  int r = foo(p, 0);
+  __builtin_free(p);
+
+  p = (char *)__builtin_malloc(1024);
+  foo(0, p);
+  __builtin_free(p);
+
+  p = (char *)__builtin_malloc(4096);
+  foo(p, 0);
+  __builtin_free(p);
+
+  // Global variable.
+  foo(&global[0], 0);
+  foo(&global[1000], 0);
+
+  p = &small_global[0];
+  foo(p, 0);
+
+  // Stack variable.
+  char stack[10000];
+  foo(&stack[0], 0);
+  foo(0, &stack[9000]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-subtract-1.c b/gcc/testsuite/c-c++-common/asan/pointer-subtract-1.c
index 7cbef811d7b..091a33f167d 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-subtract-1.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-subtract-1.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1 halt_on_error=0" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2 halt_on_error=0" } */
 /* { dg-options "-fsanitize=address,pointer-subtract" } */
 
 volatile __PTRDIFF_TYPE__ v;
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-subtract-2.c b/gcc/testsuite/c-c++-common/asan/pointer-subtract-2.c
index 6b65a16b559..7ef106c03ce 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-subtract-2.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-subtract-2.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1 halt_on_error=1" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2 halt_on_error=1" } */
 /* { dg-options "-fsanitize=address,pointer-subtract" } */
 
 volatile __PTRDIFF_TYPE__ v;
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-subtract-3.c b/gcc/testsuite/c-c++-common/asan/pointer-subtract-3.c
index 5cbcda92503..7cae91b6991 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-subtract-3.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-subtract-3.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target pthread_h } } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1:halt_on_error=1" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2:halt_on_error=1" } */
 /* { dg-options "-fsanitize=address,pointer-subtract" } */
 /* { dg-additional-options "-pthread" { target pthread } } */
 
diff --git a/gcc/testsuite/c-c++-common/asan/pointer-subtract-4.c b/gcc/testsuite/c-c++-common/asan/pointer-subtract-4.c
index 820f0aa64b0..af65c59512b 100644
--- a/gcc/testsuite/c-c++-common/asan/pointer-subtract-4.c
+++ b/gcc/testsuite/c-c++-common/asan/pointer-subtract-4.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target pthread_h } } */
 /* { dg-shouldfail "asan" } */
-/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=1:halt_on_error=1" } */
+/* { dg-set-target-env-var ASAN_OPTIONS "detect_invalid_pointer_pairs=2:halt_on_error=1" } */
 /* { dg-options "-fsanitize=address,pointer-subtract" } */
 /* { dg-additional-options "-pthread" { target pthread } } */
 
diff --git a/libsanitizer/asan/asan_flags.inc b/libsanitizer/asan/asan_flags.inc
index 3784f065be6..b4253e02524 100644
--- a/libsanitizer/asan/asan_flags.inc
+++ b/libsanitizer/asan/asan_flags.inc
@@ -134,9 +134,9 @@ ASAN_FLAG(
     "Android. ")
 ASAN_FLAG(
     int, detect_invalid_pointer_pairs, 0,
-    "If non-zero, try to detect operations like <, <=, >, >= and - on "
-    "invalid pointer pairs (e.g. when pointers belong to different objects). "
-    "The bigger the value the harder we try.")
+    "If >= 2, detect operations like <, <=, >, >= and - on invalid pointer "
+    "pairs (e.g. when pointers belong to different objects); "
+    "If == 1, detect invalid operations only when both pointers are non-null.")
 ASAN_FLAG(
     bool, detect_container_overflow, true,
     "If true, honor the container overflow annotations. See "
diff --git a/libsanitizer/asan/asan_report.cc b/libsanitizer/asan/asan_report.cc
index 261ec1aece5..434aa734c8f 100644
--- a/libsanitizer/asan/asan_report.cc
+++ b/libsanitizer/asan/asan_report.cc
@@ -340,7 +340,11 @@ static bool IsInvalidPointerPair(uptr a1, uptr a2) {
 }
 
 static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
-  if (!flags()->detect_invalid_pointer_pairs) return;
+  switch (flags()->detect_invalid_pointer_pairs) {
+    case 0 : return;
+    case 1 : if (p1 == nullptr || p2 == nullptr) return; break;
+  }
+
   uptr a1 = reinterpret_cast<uptr>(p1);
   uptr a2 = reinterpret_cast<uptr>(p2);