[PATCH][ARM][4.9 Backport] PR target/69875 Fix atomic_loaddi expansion

[Kyrill Tkachov <kyrylo dot tkachov at foss dot arm dot com>]

Hi all,

This is the GCC 4.9 backport of https://gcc.gnu.org/ml/gcc-patches/2016-02/msg01338.html.
The differences are that TARGET_HAVE_LPAE has to be defined in arm.h in a different way because
the ARM_FSET_HAS_CPU1 mechanism doesn't exist on this branch. Also, due to the location of insn_flags
and the various FL_* (on the 4.9 branch they're defined locally in arm.c rather than in arm-protos.h)
I chose to define TARGET_HAVE_LPAE in terms of hardware divide instruction availability. This should be
an equivalent definition.

Also, the scan-assembler tests that check for the DMB instruction are updated to check for
"dmb sy" rather than "dmb ish", because the memory barrier instruction changed on trunk for GCC 6.

Bootstrapped and tested on the GCC 4.9 branch on arm-none-linux-gnueabihf.

Ok for the branch after the trunk patch has had a few days to bake?

Thanks,
Kyrill

2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>

    PR target/69875
    * config/arm/arm.h (TARGET_HAVE_LPAE): Define.
    * config/arm/unspecs.md (VUNSPEC_LDRD_ATOMIC): New value.
    * config/arm/sync.md (arm_atomic_loaddi2_ldrd): New pattern.
    (atomic_loaddi_1): Delete.
    (atomic_loaddi): Rewrite expander using the above changes.

2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>

    PR target/69875
    * gcc.target/arm/atomic_loaddi_acquire.x: New file.
    * gcc.target/arm/atomic_loaddi_relaxed.x: Likewise.
    * gcc.target/arm/atomic_loaddi_seq_cst.x: Likewise.
    * gcc.target/arm/atomic_loaddi_1.c: New test.
    * gcc.target/arm/atomic_loaddi_2.c: Likewise.
    * gcc.target/arm/atomic_loaddi_3.c: Likewise.
    * gcc.target/arm/atomic_loaddi_4.c: Likewise.
    * gcc.target/arm/atomic_loaddi_5.c: Likewise.
    * gcc.target/arm/atomic_loaddi_6.c: Likewise.
    * gcc.target/arm/atomic_loaddi_7.c: Likewise.
    * gcc.target/arm/atomic_loaddi_8.c: Likewise.
    * gcc.target/arm/atomic_loaddi_9.c: Likewise.

diff --git a/gcc/config/arm/arm.h b/gcc/config/arm/arm.h
index 7c9b1b91f0926b1530bd61bb187772962b2a227d..ad97c01312cdb215768644413dd1a4061f8bd046 100644
--- a/gcc/config/arm/arm.h
+++ b/gcc/config/arm/arm.h
@@ -363,6 +363,11 @@ extern void (*arm_lang_output_object_attributes_hook)(void);
 /* Nonzero if this chip supports ldrex and strex */
 #define TARGET_HAVE_LDREX	((arm_arch6 && TARGET_ARM) || arm_arch7)
 
+/* Nonzero if this chip supports LPAE.  Such systems also support the
+   hardware divide instructions.  */
+#define TARGET_HAVE_LPAE						\
+  (arm_arch7 && arm_arch_arm_hwdiv && arm_arch_thumb_hwdiv)
+
 /* Nonzero if this chip supports ldrex{bh} and strex{bh}.  */
 #define TARGET_HAVE_LDREXBH	((arm_arch6k && TARGET_ARM) || arm_arch7)
 
diff --git a/gcc/config/arm/sync.md b/gcc/config/arm/sync.md
index 25ed926fc4e71db499d7b7fe40dc54a02182d6cb..5218ea3b8cbfee1f2fb04dd8720a1810317976af 100644
--- a/gcc/config/arm/sync.md
+++ b/gcc/config/arm/sync.md
@@ -103,31 +103,61 @@ (define_insn "atomic_store<mode>"
   [(set_attr "predicable" "yes")
    (set_attr "predicable_short_it" "no")])
 
-;; Note that ldrd and vldr are *not* guaranteed to be single-copy atomic,
-;; even for a 64-bit aligned address.  Instead we use a ldrexd unparied
-;; with a store.
+;; An LDRD instruction usable by the atomic_loaddi expander on LPAE targets
+
+(define_insn "arm_atomic_loaddi2_ldrd"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec_volatile:DI
+	[(match_operand:DI 1 "arm_sync_memory_operand" "Q")]
+	  VUNSPEC_LDRD_ATOMIC))]
+  "ARM_DOUBLEWORD_ALIGN && TARGET_HAVE_LPAE"
+  "ldrd%?\t%0, %H0, %C1"
+  [(set_attr "predicable" "yes")
+   (set_attr "predicable_short_it" "no")])
+
+;; There are three ways to expand this depending on the architecture
+;; features available.  As for the barriers, a load needs a barrier
+;; after it on all non-relaxed memory models except when the load
+;; has acquire semantics (for ARMv8-A).
+
 (define_expand "atomic_loaddi"
   [(match_operand:DI 0 "s_register_operand")		;; val out
    (match_operand:DI 1 "mem_noofs_operand")		;; memory
    (match_operand:SI 2 "const_int_operand")]		;; model
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
+  "(TARGET_HAVE_LDREXD || TARGET_HAVE_LPAE || TARGET_HAVE_LDACQ)
+   && ARM_DOUBLEWORD_ALIGN"
 {
   enum memmodel model = (enum memmodel) INTVAL (operands[2]);
-  expand_mem_thread_fence (model);
-  emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
-  if (model == MEMMODEL_SEQ_CST)
-    expand_mem_thread_fence (model);
-  DONE;
-})
 
-(define_insn "atomic_loaddi_1"
-  [(set (match_operand:DI 0 "s_register_operand" "=r")
-	(unspec:DI [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
-		   UNSPEC_LL))]
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
-  "ldrexd%?\t%0, %H0, %C1"
-  [(set_attr "predicable" "yes")
-   (set_attr "predicable_short_it" "no")])
+  /* For ARMv8-A we can use an LDAEXD to atomically load two 32-bit registers
+     when acquire or stronger semantics are needed.  When the relaxed model is
+     used this can be relaxed to a normal LDRD.  */
+  if (TARGET_HAVE_LDACQ)
+    {
+      if (model == MEMMODEL_RELAXED)
+       emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+      else
+       emit_insn (gen_arm_load_acquire_exclusivedi (operands[0], operands[1]));
+
+      DONE;
+    }
+
+  /* On LPAE targets LDRD and STRD accesses to 64-bit aligned
+     locations are 64-bit single-copy atomic.  We still need barriers in the
+     appropriate places to implement the ordering constraints.  */
+  if (TARGET_HAVE_LPAE)
+    emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+  else
+    emit_insn (gen_arm_load_exclusivedi (operands[0], operands[1]));
+
+
+  /* All non-relaxed models need a barrier after the load when load-acquire
+     instructions are not available.  */
+  if (model != MEMMODEL_RELAXED)
+     expand_mem_thread_fence (model);
+
+   DONE;
+})
 
 (define_expand "atomic_compare_and_swap<mode>"
   [(match_operand:SI 0 "s_register_operand" "")		;; bool out
diff --git a/gcc/config/arm/unspecs.md b/gcc/config/arm/unspecs.md
index 8caa953bcb9a276b540016616193c431f29e14d5..18a9a903e29e3474903ce9fec98008a57044af29 100644
--- a/gcc/config/arm/unspecs.md
+++ b/gcc/config/arm/unspecs.md
@@ -138,6 +138,7 @@ (define_c_enum "unspecv" [
   VUNSPEC_ATOMIC_XCHG	; Represent an atomic exchange.
   VUNSPEC_ATOMIC_OP	; Represent an atomic operation.
   VUNSPEC_LL		; Represent a load-register-exclusive.
+  VUNSPEC_LDRD_ATOMIC	; Represent an LDRD used as an atomic DImode load.
   VUNSPEC_SC		; Represent a store-register-exclusive.
   VUNSPEC_LAX		; Represent a load-register-acquire-exclusive.
   VUNSPEC_SLX		; Represent a store-register-release-exclusive.
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..816d352bd6b077fac38e82b54ba22796a54b3c18
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c
new file mode 100644
index 0000000000000000000000000000000000000000..c54ac7743c994b0a48e825f6e03d0fd83f0bee6e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c
new file mode 100644
index 0000000000000000000000000000000000000000..095c95889d44b889467254d3a214151a60d2de9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c
new file mode 100644
index 0000000000000000000000000000000000000000..53667b4b665ca08a6b6fa649912e51e395996336
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c
new file mode 100644
index 0000000000000000000000000000000000000000..2b9800bd2816f26caa853e7b21ae021e4dc1cedd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c
new file mode 100644
index 0000000000000000000000000000000000000000..e37ca0b280cd6ee5a13ce228a07ab381050ebd9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c
new file mode 100644
index 0000000000000000000000000000000000000000..e6deb086d60dfb339915a4e6e763148cbb73227f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c
new file mode 100644
index 0000000000000000000000000000000000000000..65530dd8b14ecf85c12d84e1b7adaf17e7d208c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tsy" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c b/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c
new file mode 100644
index 0000000000000000000000000000000000000000..3401a8eb686db3133af23ef873496ea45dd53733
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tsy" } } */
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x
new file mode 100644
index 0000000000000000000000000000000000000000..28997ef565be7640531808579e5930b61c6b50e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_acquire);
+  return glob;
+}
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x
new file mode 100644
index 0000000000000000000000000000000000000000..701b3c42c09b6bcd0add779c29321674921a95e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_relaxed);
+  return glob;
+}
diff --git a/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x b/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x
new file mode 100644
index 0000000000000000000000000000000000000000..32e78da67e8b67f8a3bdd80b4957f55d9f413669
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x
@@ -0,0 +1,11 @@
+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_seq_cst);
+  return glob;
+}