[PATCH][ARM][5 Backport] PR target/69875 Fix atomic_loaddi expansion

[Kyrill Tkachov <kyrylo dot tkachov at foss dot arm dot com>]

Hi all,

This is the GCC 5 backport of https://gcc.gnu.org/ml/gcc-patches/2016-02/msg01338.html.
The differences are that TARGET_HAVE_LPAE has to be defined in arm.h in a different way because
the ARM_FSET_HAS_CPU1 mechanism doesn't exist on this branch.
Also, the scan-assembler tests that check for the DMB instruction are updated to check for
"dmb sy" rather than "dmb ish", because the memory barrier instruction changed on trunk for GCC 6.

Bootstrapped and tested on the GCC 5 branch on arm-none-linux-gnueabihf.

Ok for the branch after the trunk patch has had a few days to bake?

Thanks,
Kyrill

2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>

    PR target/69875
    * config/arm/arm.h (TARGET_HAVE_LPAE): Define.
    * config/arm/unspecs.md (VUNSPEC_LDRD_ATOMIC): New value.
    * config/arm/sync.md (arm_atomic_loaddi2_ldrd): New pattern.
    (atomic_loaddi_1): Delete.
    (atomic_loaddi): Rewrite expander using the above changes.

2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>

    PR target/69875
    * gcc.target/arm/atomic_loaddi_acquire.x: New file.
    * gcc.target/arm/atomic_loaddi_relaxed.x: Likewise.
    * gcc.target/arm/atomic_loaddi_seq_cst.x: Likewise.
    * gcc.target/arm/atomic_loaddi_1.c: New test.
    * gcc.target/arm/atomic_loaddi_2.c: Likewise.
    * gcc.target/arm/atomic_loaddi_3.c: Likewise.
    * gcc.target/arm/atomic_loaddi_4.c: Likewise.
    * gcc.target/arm/atomic_loaddi_5.c: Likewise.
    * gcc.target/arm/atomic_loaddi_6.c: Likewise.
    * gcc.target/arm/atomic_loaddi_7.c: Likewise.
    * gcc.target/arm/atomic_loaddi_8.c: Likewise.
    * gcc.target/arm/atomic_loaddi_9.c: Likewise.

diff --git a/gcc/config/arm/arm.h b/gcc/config/arm/arm.h
index 928d18faeff645b598b5a7c678d8c8d6e35393e0..5561e433b2929df311aec3263033c14b366836d3 100644
--- a/gcc/config/arm/arm.h
+++ b/gcc/config/arm/arm.h
@@ -369,6 +369,10 @@ extern void (*arm_lang_output_object_attributes_hook)(void);
 /* Nonzero if this chip supports ldrex and strex */
 #define TARGET_HAVE_LDREX	((arm_arch6 && TARGET_ARM) || arm_arch7)
 
+/* Nonzero if this chip supports LPAE.  */
+#define TARGET_HAVE_LPAE						\
+  (arm_arch7 && ((insn_flags & FL_FOR_ARCH7VE) == FL_FOR_ARCH7VE))
+
 /* Nonzero if this chip supports ldrex{bh} and strex{bh}.  */
 #define TARGET_HAVE_LDREXBH	((arm_arch6k && TARGET_ARM) || arm_arch7)
 
diff --git a/gcc/config/arm/sync.md b/gcc/config/arm/sync.md
index 75dd52ea3aa94a227c62b6f77ee78ebf0eee61d5..acafd0a6ec474466ff7e2c67ae16d9a0dbb9cf5c 100644
--- a/gcc/config/arm/sync.md
+++ b/gcc/config/arm/sync.md
@@ -99,32 +99,62 @@ (define_insn "atomic_store<mode>"
   [(set_attr "predicable" "yes")
    (set_attr "predicable_short_it" "no")])
 
-;; Note that ldrd and vldr are *not* guaranteed to be single-copy atomic,
-;; even for a 64-bit aligned address.  Instead we use a ldrexd unparied
-;; with a store.
+;; An LDRD instruction usable by the atomic_loaddi expander on LPAE targets
+
+(define_insn "arm_atomic_loaddi2_ldrd"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(unspec_volatile:DI
+	  [(match_operand:DI 1 "arm_sync_memory_operand" "Q")]
+	    VUNSPEC_LDRD_ATOMIC))]
+  "ARM_DOUBLEWORD_ALIGN && TARGET_HAVE_LPAE"
+  "ldrd%?\t%0, %H0, %C1"
+  [(set_attr "predicable" "yes")
+   (set_attr "predicable_short_it" "no")])
+
+;; There are three ways to expand this depending on the architecture
+;; features available.  As for the barriers, a load needs a barrier
+;; after it on all non-relaxed memory models except when the load
+;; has acquire semantics (for ARMv8-A).
+
 (define_expand "atomic_loaddi"
   [(match_operand:DI 0 "s_register_operand")		;; val out
    (match_operand:DI 1 "mem_noofs_operand")		;; memory
    (match_operand:SI 2 "const_int_operand")]		;; model
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
+  "(TARGET_HAVE_LDREXD || TARGET_HAVE_LPAE || TARGET_HAVE_LDACQ)
+   && ARM_DOUBLEWORD_ALIGN"
 {
-  enum memmodel model = memmodel_from_int (INTVAL (operands[2]));
-  expand_mem_thread_fence (model);
-  emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
-  if (is_mm_seq_cst (model))
+  memmodel model = memmodel_from_int (INTVAL (operands[2]));
+
+  /* For ARMv8-A we can use an LDAEXD to atomically load two 32-bit registers
+     when acquire or stronger semantics are needed.  When the relaxed model is
+     used this can be relaxed to a normal LDRD.  */
+  if (TARGET_HAVE_LDACQ)
+    {
+      if (is_mm_relaxed (model))
+	emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+      else
+	emit_insn (gen_arm_load_acquire_exclusivedi (operands[0], operands[1]));
+
+      DONE;
+    }
+
+  /* On LPAE targets LDRD and STRD accesses to 64-bit aligned
+     locations are 64-bit single-copy atomic.  We still need barriers in the
+     appropriate places to implement the ordering constraints.  */
+  if (TARGET_HAVE_LPAE)
+    emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+  else
+    emit_insn (gen_arm_load_exclusivedi (operands[0], operands[1]));
+
+
+  /* All non-relaxed models need a barrier after the load when load-acquire
+     instructions are not available.  */
+  if (!is_mm_relaxed (model))
     expand_mem_thread_fence (model);
+
   DONE;
 })
 
-(define_insn "atomic_loaddi_1"
-  [(set (match_operand:DI 0 "s_register_operand" "=r")
-	(unspec:DI [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
-		   UNSPEC_LL))]
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
-  "ldrexd%?\t%0, %H0, %C1"
-  [(set_attr "predicable" "yes")
-   (set_attr "predicable_short_it" "no")])
-
 (define_expand "atomic_compare_and_swap<mode>"
   [(match_operand:SI 0 "s_register_operand" "")		;; bool out
    (match_operand:QHSD 1 "s_register_operand" "")	;; val out
diff --git a/gcc/config/arm/unspecs.md b/gcc/config/arm/unspecs.md
index 0ec2c48abea746df7c8006a82b4e9c43ae3ebd86..46f0089bcab53da0a72e824bd1ef9992b6cb9e54 100644
--- a/gcc/config/arm/unspecs.md
+++ b/gcc/config/arm/unspecs.md
@@ -138,6 +138,7 @@ (define_c_enum "unspecv" [
   VUNSPEC_ATOMIC_XCHG	; Represent an atomic exchange.
   VUNSPEC_ATOMIC_OP	; Represent an atomic operation.
   VUNSPEC_LL		; Represent a load-register-exclusive.
+  VUNSPEC_LDRD_ATOMIC	; Represent an LDRD used as an atomic DImode load.
   VUNSPEC_SC		; Represent a store-register-exclusive.
   VUNSPEC_LAX		; Represent a load-register-acquire-exclusive.
   VUNSPEC_SLX		; Represent a store-register-release-exclusive.
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..816d352bd6b077fac38e82b54ba22796a54b3c18
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c
new file mode 100644
index 0000000000000000000000000000000000000000..c54ac7743c994b0a48e825f6e03d0fd83f0bee6e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c
new file mode 100644
index 0000000000000000000000000000000000000000..095c95889d44b889467254d3a214151a60d2de9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c
new file mode 100644
index 0000000000000000000000000000000000000000..53667b4b665ca08a6b6fa649912e51e395996336
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c
new file mode 100644
index 0000000000000000000000000000000000000000..2b9800bd2816f26caa853e7b21ae021e4dc1cedd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c
new file mode 100644
index 0000000000000000000000000000000000000000..e37ca0b280cd6ee5a13ce228a07ab381050ebd9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c
new file mode 100644
index 0000000000000000000000000000000000000000..e6deb086d60dfb339915a4e6e763148cbb73227f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c
new file mode 100644
index 0000000000000000000000000000000000000000..65530dd8b14ecf85c12d84e1b7adaf17e7d208c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c
new file mode 100644
index 0000000000000000000000000000000000000000..3401a8eb686db3133af23ef873496ea45dd53733
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x
new file mode 100644
index 0000000000000000000000000000000000000000..28997ef565be7640531808579e5930b61c6b50e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_acquire);
+  return glob;
+}
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x
new file mode 100644
index 0000000000000000000000000000000000000000..701b3c42c09b6bcd0add779c29321674921a95e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_relaxed);
+  return glob;
+}
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x
new file mode 100644
index 0000000000000000000000000000000000000000..32e78da67e8b67f8a3bdd80b4957f55d9f413669
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_seq_cst);
+  return glob;
+}