[gcc r16-7057] middle-end: teach convert_mult_to_fma handle casts between addend and multiplicant [PR122749]
Tamar Christina
tnfchris@gcc.gnu.org
Tue Jan 27 09:13:29 GMT 2026
https://gcc.gnu.org/g:cb40e813b8f09f9d3a6000901f1373b476a20886
commit r16-7057-gcb40e813b8f09f9d3a6000901f1373b476a20886
Author: Tamar Christina <tamar.christina@arm.com>
Date: Tue Jan 27 09:12:16 2026 +0000
middle-end: teach convert_mult_to_fma handle casts between addend and multiplicant [PR122749]
The following example
int foo2 (char *buf, int len) {
int x;
for (int i =0; i < len; i++) {
x += (int) i * buf[i];
}
return x;
}
compiled with -O3 -mcpu=neoverse-v2 used to generate a 4x unrolled MLA sequence
mla z29.s, p7/m, z2.s, z0.s
mla z27.s, p7/m, z4.s, z26.s
mla z30.s, p7/m, z1.s, z0.s
mla z28.s, p7/m, z23.s, z3.s
but now generates MUL + ADD
mul z2.s, z2.s, z1.s
mul z4.s, z4.s, z26.s
mul z1.s, z24.s, z1.s
mul z3.s, z23.s, z3.s
add z29.s, z2.s, z29.s
add z30.s, z1.s, z30.s
add z28.s, z3.s, z28.s
add z0.s, z4.s, z0.s
This is since the fix for r16-3328-g3182e95eda4 we now insert casts around the
reduction addend. This causes convert_mult_to_fma to miss the mul + add
sequence.
This patch teaches it to look around the casts for the operands and only accept
the conversions if it's essentially only a sign changing operations.
Concretely, it converts:
# vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)>
vect__3.8_38 = MEM <vector([4,4]) char> [(char *)_16];
vect__4.12_45 = (vector([4,4]) int) vect__3.8_38;
vect__5.14_54 = vect__4.12_45 * vect_vec_iv_.13_49;
vect_x_12.17_62 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect__5.14_54);
vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58);
vect_x_12.17_64 = vect_x_12.17_62 + vect_x_12.17_63;
vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) int>(vect_x_12.17_64);
into:
# vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)>
vect__3.8_38 = MEM <vector([4,4]) charD.8> [(charD.8 *)_16];
vect__4.12_45 = (vector([4,4]) intD.7) vect__3.8_38;
vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58);
_2 = (vector([4,4]) unsigned int) vect_vec_iv_.13_49;
_1 = (vector([4,4]) unsigned int) vect__4.12_45;
vect_x_12.17_64 = .FMA (_1, _2, vect_x_12.17_63);
vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) intD.7>(vect_x_12.17_64);
thus restoring FMAs on reductions.
gcc/ChangeLog:
PR tree-optimization/122749
* tree-ssa-math-opts.cc (convert_mult_to_fma_1, convert_mult_to_fma):
Unwrap converts around addend.
gcc/testsuite/ChangeLog:
PR tree-optimization/122749
* gcc.target/aarch64/pr122749_1.c: New test.
* gcc.target/aarch64/pr122749_2.c: New test.
* gcc.target/aarch64/pr122749_3.c: New test.
* gcc.target/aarch64/pr122749_4.c: New test.
* gcc.target/aarch64/pr122749_5.c: New test.
* gcc.target/aarch64/pr122749_6.c: New test.
* gcc.target/aarch64/pr122749_8.c: New test.
* gcc.target/aarch64/pr122749_9.c: New test.
* gcc.target/aarch64/sve/pr122749_1.c: New test.
* gcc.target/aarch64/sve/pr122749_11.c: New test.
* gcc.target/aarch64/sve/pr122749_12.c: New test.
* gcc.target/aarch64/sve/pr122749_13.c: New test.
* gcc.target/aarch64/sve/pr122749_14.c: New test.
* gcc.target/aarch64/sve/pr122749_2.c: New test.
* gcc.target/aarch64/sve/pr122749_3.c: New test.
* gcc.target/aarch64/sve/pr122749_4.c: New test.
* gcc.target/aarch64/sve/pr122749_5.c: New test.
* gcc.target/aarch64/sve/pr122749_6.c: New test.
* gcc.target/aarch64/sve/pr122749_8.c: New test.
* gcc.target/aarch64/sve/pr122749_9.c: New test.
Diff:
---
gcc/testsuite/gcc.target/aarch64/pr122749_1.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_2.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_3.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_4.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_5.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_6.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_8.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/pr122749_9.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c | 48 ++++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c | 45 ++++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c | 45 ++++++++++++++++++++
gcc/tree-ssa-math-opts.cc | 41 ++++++++++++++++++
21 files changed, 980 insertions(+)
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_1.c b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c
new file mode 100644
index 000000000000..25311fce4e3a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_1.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_2.c b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c
new file mode 100644
index 000000000000..f4a70a611176
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_2.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_3.c b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c
new file mode 100644
index 000000000000..61bcd30be2b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_3.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_4.c b/gcc/testsuite/gcc.target/aarch64/pr122749_4.c
new file mode 100644
index 000000000000..6089716b0ca7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_4.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_5.c b/gcc/testsuite/gcc.target/aarch64/pr122749_5.c
new file mode 100644
index 000000000000..562dc5be8617
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_5.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 2 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_6.c b/gcc/testsuite/gcc.target/aarch64/pr122749_6.c
new file mode 100644
index 000000000000..3e51c5e22a18
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_6.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_8.c b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c
new file mode 100644
index 000000000000..6aa729c13d16
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_8.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr122749_9.c b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c
new file mode 100644
index 000000000000..d987a9936afb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr122749_9.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c
new file mode 100644
index 000000000000..32a36461fbc7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_1.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c
new file mode 100644
index 000000000000..bd160dd0ebf5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_11.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c
new file mode 100644
index 000000000000..8f0198ce4260
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_12.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c
new file mode 100644
index 000000000000..218afde13984
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_13.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT32_MAX, 7, 0, UINT32_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c
new file mode 100644
index 000000000000..1587628757e2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_14.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint64_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT64_MAX, 7, 0, UINT64_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c
new file mode 100644
index 000000000000..0f5918a90235
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_2.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c
new file mode 100644
index 000000000000..92548cb6ec4f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_3.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c
new file mode 100644
index 000000000000..6085a18bab7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_4.c
@@ -0,0 +1,48 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int64_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT64_MAX, INT64_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c
new file mode 100644
index 000000000000..d61b91bb06dc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_5.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c
new file mode 100644
index 000000000000..7598f7a28bcf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_6.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c
new file mode 100644
index 000000000000..e1c337d44ead
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_8.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c
new file mode 100644
index 000000000000..13d962e2130f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr122749_9.c
@@ -0,0 +1,45 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc
index 4c3fb0f4fc53..4b50a96ad3aa 100644
--- a/gcc/tree-ssa-math-opts.cc
+++ b/gcc/tree-ssa-math-opts.cc
@@ -3120,6 +3120,26 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
if (is_gimple_debug (use_stmt))
continue;
+ /* If the use is a type convert, look further into it if the operations
+ are the same under two's complement. */
+ tree lhs_type;
+ if (gimple_assign_cast_p (use_stmt)
+ && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
+ && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
+ {
+ tree cast_lhs = gimple_get_lhs (use_stmt);
+ gimple *tmp_use;
+ use_operand_p tmp_use_p;
+ if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
+ {
+ release_defs (use_stmt);
+ use_stmt = tmp_use;
+ result = cast_lhs;
+ gsi_remove (&gsi, true);
+ gsi = gsi_for_stmt (use_stmt);
+ }
+ }
+
if (is_gimple_assign (use_stmt)
&& gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
{
@@ -3159,6 +3179,13 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
if (seq)
gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
+ /* Ensure all the operands are of the same type. Use the type of the
+ addend as that's the statement being replaced. */
+ op2 = gimple_convert (&gsi, true, GSI_SAME_STMT,
+ UNKNOWN_LOCATION, TREE_TYPE (addop), op2);
+ mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT,
+ UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1);
+
if (len)
fma_stmt
= gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2,
@@ -3419,6 +3446,20 @@ convert_mult_to_fma (gimple *mul_stmt, tree op1, tree op2,
if (is_gimple_debug (use_stmt))
continue;
+ /* If the use is a type convert, look further into it if the operations
+ are the same under two's complement. */
+ tree lhs_type;
+ if (gimple_assign_cast_p (use_stmt)
+ && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
+ && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
+ {
+ tree cast_lhs = gimple_get_lhs (use_stmt);
+ gimple *tmp_use;
+ use_operand_p tmp_use_p;
+ if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
+ use_stmt = tmp_use;
+ }
+
/* For now restrict this operations to single basic blocks. In theory
we would want to support sinking the multiplication in
m = a*b;
More information about the Gcc-cvs
mailing list