[gcc(refs/users/mikael/heads/refactor_descriptor_v08)] Régénération fichiers générés
Mikael Morin
mikael@gcc.gnu.org
Fri Sep 12 16:04:27 GMT 2025
https://gcc.gnu.org/g:75bb7960cb32e3540a19e1846e57ddf1391594dd
commit 75bb7960cb32e3540a19e1846e57ddf1391594dd
Author: Mikael Morin <mikael@gcc.gnu.org>
Date: Fri Sep 12 17:44:58 2025 +0200
Régénération fichiers générés
Régénération fichiers générés
Régénération fichiers générés
Diff:
---
libgfortran/generated/matmul_c10.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_c16.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_c17.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_c4.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_c8.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_i1.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_i16.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_i2.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_i4.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_i8.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_r10.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_r16.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_r17.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_r4.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmul_r8.c | 1115 +++++++++++++++++-------------
libgfortran/generated/matmulavx128_c10.c | 446 ++++++------
libgfortran/generated/matmulavx128_c16.c | 446 ++++++------
libgfortran/generated/matmulavx128_c17.c | 446 ++++++------
libgfortran/generated/matmulavx128_c4.c | 446 ++++++------
libgfortran/generated/matmulavx128_c8.c | 446 ++++++------
libgfortran/generated/matmulavx128_i1.c | 446 ++++++------
libgfortran/generated/matmulavx128_i16.c | 446 ++++++------
libgfortran/generated/matmulavx128_i2.c | 446 ++++++------
libgfortran/generated/matmulavx128_i4.c | 446 ++++++------
libgfortran/generated/matmulavx128_i8.c | 446 ++++++------
libgfortran/generated/matmulavx128_r10.c | 446 ++++++------
libgfortran/generated/matmulavx128_r16.c | 446 ++++++------
libgfortran/generated/matmulavx128_r17.c | 446 ++++++------
libgfortran/generated/matmulavx128_r4.c | 446 ++++++------
libgfortran/generated/matmulavx128_r8.c | 446 ++++++------
30 files changed, 13020 insertions(+), 10395 deletions(-)
diff --git a/libgfortran/generated/matmul_c10.c b/libgfortran/generated/matmul_c10.c
index 3193126ce228..7490e041f143 100644
--- a/libgfortran/generated/matmul_c10.c
+++ b/libgfortran/generated/matmul_c10.c
@@ -94,6 +94,8 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_10);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_10 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_10 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_10) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_10) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_10) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_10)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_10)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_c10_avx (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_10);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_10 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_10 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_10) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_10) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_10) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_10)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_10)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_c10_avx2 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_10);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_10 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_10 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_10) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_10) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_10) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_10)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_10)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_c10_avx512f (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_10);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_10 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_10 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_10) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_10) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_10) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_10)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_10)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_c10_vanilla (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_10);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_10 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_10 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_10) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_10) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_10) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_10)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_10)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_c10 (gfc_array_c10 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_10) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
diff --git a/libgfortran/generated/matmul_c16.c b/libgfortran/generated/matmul_c16.c
index 33c495ae7df5..d3ba317cc5fd 100644
--- a/libgfortran/generated/matmul_c16.c
+++ b/libgfortran/generated/matmul_c16.c
@@ -94,6 +94,8 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_16);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_16 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_16 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_16) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_16) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_16) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_16)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_16)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_c16_avx (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_16);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_16 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_16 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_16) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_16) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_16) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_16)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_16)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_c16_avx2 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_16);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_16 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_16 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_16) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_16) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_16) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_16)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_16)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_c16_avx512f (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_16);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_16 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_16 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_16) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_16) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_16) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_16)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_16)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_c16_vanilla (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_16);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_16 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_16 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_16) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_16) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_16) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_16)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_16)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_c16 (gfc_array_c16 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_16) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
diff --git a/libgfortran/generated/matmul_c17.c b/libgfortran/generated/matmul_c17.c
index 39cd2e0308c1..82ab0ed19362 100644
--- a/libgfortran/generated/matmul_c17.c
+++ b/libgfortran/generated/matmul_c17.c
@@ -94,6 +94,8 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_17);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_17 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_17 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_17) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_17) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_17) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_17)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_17)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_c17_avx (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_17);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_17 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_17 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_17) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_17) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_17) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_17)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_17)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_c17_avx2 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_17);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_17 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_17 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_17) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_17) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_17) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_17)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_17)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_c17_avx512f (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_17);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_17 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_17 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_17) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_17) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_17) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_17)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_17)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_c17_vanilla (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_17);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_17 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_17 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_17) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_17) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_17) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_17)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_17)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_c17 (gfc_array_c17 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_17) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
diff --git a/libgfortran/generated/matmul_c4.c b/libgfortran/generated/matmul_c4.c
index 170081708deb..a7c622888d0b 100644
--- a/libgfortran/generated/matmul_c4.c
+++ b/libgfortran/generated/matmul_c4.c
@@ -94,6 +94,8 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_4);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_4 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_4 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_4) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_4) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_4) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_4)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_4)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_c4_avx (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_4);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_4 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_4 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_4) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_4) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_4) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_4)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_4)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_c4_avx2 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_4);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_4 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_4 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_4) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_4) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_4) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_4)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_4)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_c4_avx512f (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_4);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_4 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_4 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_4) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_4) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_4) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_4)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_4)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_c4_vanilla (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_4);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_4 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_4 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_4) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_4) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_4) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_4)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_4)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_c4 (gfc_array_c4 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_4) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
diff --git a/libgfortran/generated/matmul_c8.c b/libgfortran/generated/matmul_c8.c
index debf5c3f5155..ec7a48f1d53a 100644
--- a/libgfortran/generated/matmul_c8.c
+++ b/libgfortran/generated/matmul_c8.c
@@ -94,6 +94,8 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_8);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_8 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_8 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_8) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_8) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_8) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_8)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_8)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_c8_avx (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_8);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_8 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_8 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_8) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_8) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_8) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_8)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_8)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_c8_avx2 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_8);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_8 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_8 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_8) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_8) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_8) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_8)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_8)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_c8_avx512f (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_8);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_8 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_8 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_8) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_8) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_8) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_8)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_8)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_c8_vanilla (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_COMPLEX_8);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_COMPLEX_8 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_COMPLEX_8 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_COMPLEX_8) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_COMPLEX_8) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_COMPLEX_8) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_COMPLEX_8)0;
+ C_ARRAY_ELEM (i, j) = (GFC_COMPLEX_8)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_c8 (gfc_array_c8 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_COMPLEX_8) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
diff --git a/libgfortran/generated/matmul_i1.c b/libgfortran/generated/matmul_i1.c
index 8f7a63bb8eb8..aad06c66344c 100644
--- a/libgfortran/generated/matmul_i1.c
+++ b/libgfortran/generated/matmul_i1.c
@@ -94,6 +94,8 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -182,11 +184,14 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -194,6 +199,8 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_UINTEGER_1);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -202,6 +209,8 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -219,17 +228,21 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -288,6 +301,7 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_UINTEGER_1 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -301,14 +315,25 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_UINTEGER_1 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_UINTEGER_1) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_UINTEGER_1) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_UINTEGER_1) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_UINTEGER_1)0;
+ C_ARRAY_ELEM (i, j) = (GFC_UINTEGER_1)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -360,20 +385,20 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -382,7 +407,7 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -393,100 +418,100 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -498,38 +523,38 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -538,6 +563,9 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -550,11 +578,11 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -569,12 +597,11 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -585,12 +612,11 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -617,17 +643,16 @@ matmul_i1_avx (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -654,6 +679,8 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -742,11 +769,14 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -754,6 +784,8 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_UINTEGER_1);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -762,6 +794,8 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -779,17 +813,21 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -848,6 +886,7 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_UINTEGER_1 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -861,14 +900,25 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_UINTEGER_1 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_UINTEGER_1) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_UINTEGER_1) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_UINTEGER_1) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_UINTEGER_1)0;
+ C_ARRAY_ELEM (i, j) = (GFC_UINTEGER_1)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -920,20 +970,20 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -942,7 +992,7 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -953,100 +1003,100 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1058,38 +1108,38 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1098,6 +1148,9 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1110,11 +1163,11 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1129,12 +1182,11 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1145,12 +1197,11 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1177,17 +1228,16 @@ matmul_i1_avx2 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1214,6 +1264,8 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1302,11 +1354,14 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1314,6 +1369,8 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_UINTEGER_1);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1322,6 +1379,8 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1339,17 +1398,21 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1408,6 +1471,7 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_UINTEGER_1 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1421,14 +1485,25 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_UINTEGER_1 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_UINTEGER_1) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_UINTEGER_1) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_UINTEGER_1) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_UINTEGER_1)0;
+ C_ARRAY_ELEM (i, j) = (GFC_UINTEGER_1)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -1480,20 +1555,20 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -1502,7 +1577,7 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -1513,100 +1588,100 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -1618,38 +1693,38 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -1658,6 +1733,9 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -1670,11 +1748,11 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -1689,12 +1767,11 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -1705,12 +1782,11 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -1737,17 +1813,16 @@ matmul_i1_avx512f (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -1788,6 +1863,8 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -1876,11 +1953,14 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -1888,6 +1968,8 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_UINTEGER_1);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1896,6 +1978,8 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -1913,17 +1997,21 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -1982,6 +2070,7 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_UINTEGER_1 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -1995,14 +2084,25 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_UINTEGER_1 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_UINTEGER_1) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_UINTEGER_1) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_UINTEGER_1) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_UINTEGER_1)0;
+ C_ARRAY_ELEM (i, j) = (GFC_UINTEGER_1)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2054,20 +2154,20 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2076,7 +2176,7 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2087,100 +2187,100 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2192,38 +2292,38 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2232,6 +2332,9 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2244,11 +2347,11 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2263,12 +2366,11 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2279,12 +2381,11 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2311,17 +2412,16 @@ matmul_i1_vanilla (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 0, x)) = s;
+ s += ARRAY_ELEM_AT_OFFSET (abase_x, n * aystride_bytes)
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
+ ARRAY_ELEM_AT_OFFSET (dest_y, x * rxstride_bytes) = s;
}
}
}
@@ -2421,6 +2521,8 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
index_type rxstride, rystride, axstride, aystride, bxstride, bystride;
index_type x, y, n, count, xcount, ycount;
+ index_type axstride_bytes, aystride_bytes, bxstride_bytes, bystride_bytes,
+ rxstride_bytes, rystride_bytes;
assert (GFC_DESCRIPTOR_RANK (a) == 2
|| GFC_DESCRIPTOR_RANK (b) == 2);
@@ -2509,11 +2611,14 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
either as a row or a column matrix. We want both cases to
work. */
rxstride = rystride = GFC_DESCRIPTOR_STRIDE(retarray,0);
+ rxstride_bytes = rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
}
else
{
rxstride = GFC_DESCRIPTOR_STRIDE(retarray,0);
rystride = GFC_DESCRIPTOR_STRIDE(retarray,1);
+ rxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,0);
+ rystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(retarray,1);
}
if (GFC_DESCRIPTOR_RANK (a) == 1)
@@ -2521,6 +2626,8 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
/* Treat it as a a row matrix A[1,count]. */
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = 1;
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = sizeof (GFC_UINTEGER_1);
xcount = 1;
count = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2529,6 +2636,8 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
{
axstride = GFC_DESCRIPTOR_STRIDE(a,0);
aystride = GFC_DESCRIPTOR_STRIDE(a,1);
+ axstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,0);
+ aystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(a,1);
count = GFC_DESCRIPTOR_EXTENT(a,1);
xcount = GFC_DESCRIPTOR_EXTENT(a,0);
@@ -2546,17 +2655,21 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
{
/* Treat it as a column matrix B[count,1] */
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
/* bystride should never be used for 1-dimensional b.
The value is only used for calculation of the
memory by the buffer. */
bystride = 256;
+ bystride_bytes = 99999999;
ycount = 1;
}
else
{
bxstride = GFC_DESCRIPTOR_STRIDE(b,0);
bystride = GFC_DESCRIPTOR_STRIDE(b,1);
+ bxstride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,0);
+ bystride_bytes = GFC_DESCRIPTOR_STRIDE_BYTES(b,1);
ycount = GFC_DESCRIPTOR_EXTENT(b,1);
}
@@ -2615,6 +2728,7 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
from netlib.org, translated to C, and modified for matmul.m4. */
+ GFC_UINTEGER_1 *c;
const index_type m = xcount, n = ycount, k = count;
/* System generated locals */
@@ -2628,14 +2742,25 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
index_type isec, jsec, lsec, uisec, ujsec, ulsec;
GFC_UINTEGER_1 *t1;
+ c = retarray->base_addr;
+
/* Parameter adjustments */
a_dim1 = aystride;
b_dim1 = bystride;
+#define A_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (abase, (i) * sizeof (GFC_UINTEGER_1) + (j) * aystride_bytes))
+
+#define B_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (bbase, (i) * sizeof (GFC_UINTEGER_1) + (j) * bystride_bytes))
+
+#define C_ARRAY_ELEM(i,j) \
+ (ARRAY_ELEM_AT_OFFSET (c, (i) * sizeof (GFC_UINTEGER_1) + (j) * rystride_bytes))
+
/* Empty result first. */
for (j=0; j<n; j++)
for (i=0; i<m; i++)
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = (GFC_UINTEGER_1)0;
+ C_ARRAY_ELEM (i, j) = (GFC_UINTEGER_1)0;
/* Early exit if possible */
if (m == 0 || n == 0 || k == 0)
@@ -2687,20 +2812,20 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (i = ii; i < i5; i += 2)
{
t1[l - ll + 1 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l);
+ A_ARRAY_ELEM (i, l);
t1[l - ll + 2 + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, l + 1);
+ A_ARRAY_ELEM (i, l + 1);
t1[l - ll + 1 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l);
+ A_ARRAY_ELEM (i + 1, l);
t1[l - ll + 2 + ((i - ii + 2) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i + 1, l + 1);
+ A_ARRAY_ELEM (i + 1, l + 1);
}
if (uisec < isec)
{
t1[l - ll + 1 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l);
+ A_ARRAY_ELEM (ii + isec - 1, l);
t1[l - ll + 2 + (isec << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, ii + isec - 1, l + 1);
+ A_ARRAY_ELEM (ii + isec - 1, l + 1);
}
}
if (ulsec < lsec)
@@ -2709,7 +2834,7 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (i = ii; i< i4; ++i)
{
t1[lsec + ((i - ii + 1) << 8) - 257] =
- GFC_DESCRIPTOR2_ELEM (a, i, ll + lsec - 1);
+ A_ARRAY_ELEM (i, ll + lsec - 1);
}
}
@@ -2720,100 +2845,100 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f22 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f23 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
- f24 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
- f32 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1);
- f42 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1);
- f33 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2);
- f43 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2);
- f34 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3);
- f44 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f22 = C_ARRAY_ELEM (i + 1, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f23 = C_ARRAY_ELEM (i + 1, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
+ f24 = C_ARRAY_ELEM (i + 1, j + 3);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
+ f32 = C_ARRAY_ELEM (i + 2, j + 1);
+ f42 = C_ARRAY_ELEM (i + 3, j + 1);
+ f33 = C_ARRAY_ELEM (i + 2, j + 2);
+ f43 = C_ARRAY_ELEM (i + 3, j + 2);
+ f34 = C_ARRAY_ELEM (i + 2, j + 3);
+ f44 = C_ARRAY_ELEM (i + 3, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f22 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f23 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f24 += t1[l - ll + 1 + ((i - ii + 2) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ * B_ARRAY_ELEM (l, j);
f32 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f42 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ * B_ARRAY_ELEM (l, j + 1);
f33 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f43 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ * B_ARRAY_ELEM (l, j + 2);
f34 += t1[l - ll + 1 + ((i - ii + 3) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
f44 += t1[l - ll + 1 + ((i - ii + 4) << 8) - 257]
- * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 1) = f22;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 2) = f23;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j + 3) = f24;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 1) = f32;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 1) = f42;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 2) = f33;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 2) = f43;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j + 3) = f34;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j + 3) = f44;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i + 1, j + 1) = f22;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i + 1, j + 2) = f23;
+ C_ARRAY_ELEM (i, j + 3) = f14;
+ C_ARRAY_ELEM (i + 1, j + 3) = f24;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
+ C_ARRAY_ELEM (i + 2, j + 1) = f32;
+ C_ARRAY_ELEM (i + 3, j + 1) = f42;
+ C_ARRAY_ELEM (i + 2, j + 2) = f33;
+ C_ARRAY_ELEM (i + 3, j + 2) = f43;
+ C_ARRAY_ELEM (i + 2, j + 3) = f34;
+ C_ARRAY_ELEM (i + 3, j + 3) = f44;
}
if (uisec < isec)
{
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f12 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1);
- f13 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2);
- f14 = GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3);
+ f11 = C_ARRAY_ELEM (i, j);
+ f12 = C_ARRAY_ELEM (i, j + 1);
+ f13 = C_ARRAY_ELEM (i, j + 2);
+ f14 = C_ARRAY_ELEM (i, j + 3);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f12 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 1);
+ 257] * B_ARRAY_ELEM (l, j + 1);
f13 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 2);
+ 257] * B_ARRAY_ELEM (l, j + 2);
f14 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j + 3);
+ 257] * B_ARRAY_ELEM (l, j + 3);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 1) = f12;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 2) = f13;
- GFC_DESCRIPTOR2_ELEM (retarray, i, j + 3) = f14;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i, j + 1) = f12;
+ C_ARRAY_ELEM (i, j + 2) = f13;
+ C_ARRAY_ELEM (i, j + 3) = f14;
}
}
}
@@ -2825,38 +2950,38 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
i5 = ii + uisec;
for (i = ii; i < i5; i += 4)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
- f21 = GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j);
- f31 = GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j);
- f41 = GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j);
+ f11 = C_ARRAY_ELEM (i, j);
+ f21 = C_ARRAY_ELEM (i + 1, j);
+ f31 = C_ARRAY_ELEM (i + 2, j);
+ f41 = C_ARRAY_ELEM (i + 3, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f21 += t1[l - ll + 1 + ((i - ii + 2) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f31 += t1[l - ll + 1 + ((i - ii + 3) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
f41 += t1[l - ll + 1 + ((i - ii + 4) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 1, j) = f21;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 2, j) = f31;
- GFC_DESCRIPTOR2_ELEM (retarray, i + 3, j) = f41;
+ C_ARRAY_ELEM (i, j) = f11;
+ C_ARRAY_ELEM (i + 1, j) = f21;
+ C_ARRAY_ELEM (i + 2, j) = f31;
+ C_ARRAY_ELEM (i + 3, j) = f41;
}
i5 = ii + isec;
for (i = ii + uisec; i < i5; ++i)
{
- f11 = GFC_DESCRIPTOR2_ELEM (retarray, i, j);
+ f11 = C_ARRAY_ELEM (i, j);
i6 = ll + lsec;
for (l = ll; l < i6; ++l)
{
f11 += t1[l - ll + 1 + ((i - ii + 1) << 8) -
- 257] * GFC_DESCRIPTOR2_ELEM (b, l, j);
+ 257] * B_ARRAY_ELEM (l, j);
}
- GFC_DESCRIPTOR2_ELEM (retarray, i, j) = f11;
+ C_ARRAY_ELEM (i, j) = f11;
}
}
}
@@ -2865,6 +2990,9 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
}
free(t1);
return;
+#undef A_ARRAY_ELEM
+#undef B_ARRAY_ELEM
+#undef C_ARRAY_ELEM
}
else if (rxstride == 1 && aystride == 1 && bxstride == 1)
{
@@ -2877,11 +3005,11 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (dest, GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (dest, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += abase_x[n] * bbase_y[n];
@@ -2896,12 +3024,11 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n) * bbase_y[n];
- ARRAY_ELEM_AT_OFFSET (dest,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y)) = s;
+ ARRAY_ELEM_AT_OFFSET (dest, y * rystride_bytes) = s;
}
}
}
@@ -2912,12 +3039,11 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
s += GFC_DESCRIPTOR1_ELEM (a, n)
- * ARRAY_ELEM_AT_OFFSET (bbase_y,
- GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
+ * ARRAY_ELEM_AT_OFFSET (bbase_y, n * bxstride_bytes);
GFC_DESCRIPTOR1_ELEM (retarray, y) = s;
}
}
@@ -2944,17 +3070,16 @@ matmul_i1 (gfc_array_m1 * const restrict retarray,
for (y = 0; y < ycount; y++)
{
- bbase_y = PTR_ADD_OFFSET (bbase, GFC_DESCRIPTOR_DIM_OFFSET (b, 1, y));
- dest_y = PTR_ADD_OFFSET (retarray->base_addr,
- GFC_DESCRIPTOR_DIM_OFFSET (retarray, 1, y));
+ bbase_y = PTR_ADD_OFFSET (bbase, y * bystride_bytes);
+ dest_y = PTR_ADD_OFFSET (retarray->base_addr, y * rystride_bytes);
for (x = 0; x < xcount; x++)
{
- abase_x = PTR_ADD_OFFSET (abase, GFC_DESCRIPTOR_DIM_OFFSET (a, 0, x));
+ abase_x = PTR_ADD_OFFSET (abase, x * axstride_bytes);
s = (GFC_UINTEGER_1) 0;
for (n = 0; n < count; n++)
- s += ARRAY_ELEM_AT_OFFSET (abase_x, GFC_DESCRIPTOR_DIM_OFFSET (a, 1, n))
- * ARRAY_ELEM_AT_OFFSET (bbase_y, GFC_DESCRIPTOR_DIM_OFFSET (b, 0, n));
- ARRAY_ELEM_AT_OFFSET (dest_y, GFC_DESCRIPTOR_DIM_OFFSET (retarra[...]
[diff truncated at 524288 bytes]
More information about the Gcc-cvs
mailing list