This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Re: PR81635: Use chrecs to help find related data refs
- From: Richard Sandiford <richard dot sandiford at linaro dot org>
- To: gcc-patches at gcc dot gnu dot org
- Date: Wed, 16 Aug 2017 14:40:23 +0100
- Subject: Re: PR81635: Use chrecs to help find related data refs
- Authentication-results: sourceware.org; auth=none
- References: <87pobvr5t8.fsf@linaro.org>
Richard Sandiford <richard.sandiford@linaro.org> writes:
> The first loop in the testcase regressed after my recent changes to
> dr_analyze_innermost. Previously we would treat "i" as an iv even
> for bb analysis and end up with:
>
> DR_BASE_ADDRESS: p or q
> DR_OFFSET: 0
> DR_INIT: 0 or 4
> DR_STEP: 16
>
> We now always keep the step as 0 instead, so for an int "i" we'd have:
>
> DR_BASE_ADDRESS: p or q
> DR_OFFSET: (intptr_t) i
> DR_INIT: 0 or 4
> DR_STEP: 0
>
> This is also what we'd like to have for the unsigned "i", but the
> problem is that strip_constant_offset thinks that the "i + 1" in
> "(intptr_t) (i + 1)" could wrap and so doesn't peel off the "+ 1".
> The [i + 1] accesses therefore have a DR_OFFSET equal to the SSA
> name that holds "(intptr_t) (i + 1)", meaning that the accesses no
> longer seem to be related to the [i] ones.
>
> There seem to be two main uses of DR_OFFSET and DR_INIT. One type
> expects DR_OFFSET and DR_INIT to be generic expressions whose sum
> gives the initial offset from DR_BASE_ADDRESS. The other type uses
> the pair (DR_BASE_ADDRESS, DR_OFFSET) to identify related data
> references, with the distance between their start addresses being
> the difference between their DR_INITs. For this second type, the
> exact form of DR_OFFSET doesn't really matter, and the more it is
> able to canonicalise the non-constant offset, the better.
>
> The patch fixes the PR by adding another offset/init pair to the
> innermost loop behaviour for this second group. The new pair use chrecs
> rather than generic exprs for the offset part, with the chrec being
> analysed in the innermost loop for which the offset isn't invariant.
> This allows us to vectorise the second function in the testcase
> as well, which wasn't possible before the earlier patch.
>
> Tested on x86_64-linux-gnu and aarch64-linux-gnu. OK to install?
>
> Richard
>
>
> gcc/
> PR tree-optimization/81635
> * tree-data-ref.h (innermost_loop_behavior): Add chrec_offset and
> chrec_init.
> (DR_CHREC_OFFSET, DR_CHREC_INIT): New macros.
> (dr_equal_offsets_p): Delete.
> (dr_chrec_offsets_equal_p, dr_chrec_offsets_compare): Declare.
> * tree-data-ref.c: Include tree-ssa-loop-ivopts.h.
> (split_constant_offset): Handle POLYNOMIAL_CHREC.
> (dr_analyze_innermost): Initialize dr_chrec_offset and dr_chrec_init.
> (operator ==): Use dr_chrec_offsets_equal_p and compare the
> DR_CHREC_INITs.
> (prune_runtime_alias_test_list): Likewise.
> (comp_dr_with_seg_len_pair): Use dr_chrec_offsets_compare and compare
> the DR_CHREC_INITs.
> (dr_equal_offsets_p1, dr_equal_offsets_p): Delete.
> (analyze_offset_scev): New function.
> (compute_offset_chrecs): Likewise.
> (dr_chrec_offsets_equal_p): Likewise.
> (dr_chrec_offsets_compare): Likewise.
> * tree-loop-distribution.c (compute_alias_check_pairs): Use
> dr_chrec_offsets_compare.
> * tree-vect-data-refs.c (vect_find_same_alignment_drs): Use
> dr_chrec_offsets_compare and compare the DR_CHREC_INITs.
> (dr_group_sort_cmp): Likewise.
> (vect_analyze_group_access_1): Use DR_CHREC_INIT instead of DR_INIT.
> (vect_no_alias_p): Likewise.
> (vect_analyze_data_ref_accesses): Use dr_chrec_offsets_equal_p and
> compare the DR_CHREC_INITs.
> (vect_prune_runtime_alias_test_list): Use dr_chrec_offsets_compare.
> * tree-vect-stmts.c (vectorizable_load): Use DR_CHREC_INIT instead
> of DR_INIT.
>
> gcc/testsuite/
> PR tree-optimization/81635
> * gcc.dg/vect/bb-slp-pr81635.c: New test.
Sorry, forgot I was going to convert this to a context diff...
Index: gcc/tree-data-ref.h
===================================================================
*** gcc/tree-data-ref.h 2017-08-04 11:42:45.938134820 +0100
--- gcc/tree-data-ref.h 2017-08-16 14:34:56.611554296 +0100
*************** struct innermost_loop_behavior
*** 52,57 ****
--- 52,78 ----
tree init;
tree step;
+ /* The scalar evolution of OFFSET + INIT, split into non-constant and
+ constant terms in the same way as OFFSET and INIT. For example,
+ if OFFSET + INIT is {x + 4, +, 3}_1, the fields would be:
+
+ chrec_offset {x, +, 3}_1
+ chrec_init 4
+
+ If OFFSET + INIT cannot be represented as a scalar evolution,
+ the fields are simply a copy of OFFSET and INIT.
+
+ This split is useful when analyzing the relationship between two
+ data references with the same base. OFFSET and INIT are generic
+ expressions that can be used to build related data references,
+ while CHREC_OFFSET and CHREC_INIT are our best attempt at
+ canonicalizing the value of OFFSET + INIT.
+
+ These fields are only initialized lazily, by a call to
+ compute_offset_chrecs. */
+ tree chrec_offset;
+ tree chrec_init;
+
/* BASE_ADDRESS is known to be misaligned by BASE_MISALIGNMENT bytes
from an alignment boundary of BASE_ALIGNMENT bytes. For example,
if we had:
*************** #define DR_IS_CONDITIONAL_IN_STMT(DR) (D
*** 187,192 ****
--- 208,215 ----
#define DR_BASE_ADDRESS(DR) (DR)->innermost.base_address
#define DR_OFFSET(DR) (DR)->innermost.offset
#define DR_INIT(DR) (DR)->innermost.init
+ #define DR_CHREC_OFFSET(DR) (DR)->innermost.chrec_offset
+ #define DR_CHREC_INIT(DR) (DR)->innermost.chrec_init
#define DR_STEP(DR) (DR)->innermost.step
#define DR_PTR_INFO(DR) (DR)->alias.ptr_info
#define DR_BASE_ALIGNMENT(DR) (DR)->innermost.base_alignment
*************** dr_alignment (data_reference *dr)
*** 466,473 ****
extern bool dr_may_alias_p (const struct data_reference *,
const struct data_reference *, bool);
! extern bool dr_equal_offsets_p (struct data_reference *,
! struct data_reference *);
extern bool runtime_alias_check_p (ddr_p, struct loop *, bool);
extern int data_ref_compare_tree (tree, tree);
--- 489,498 ----
extern bool dr_may_alias_p (const struct data_reference *,
const struct data_reference *, bool);
! extern bool dr_chrec_offsets_equal_p (struct data_reference *,
! struct data_reference *);
! extern int dr_chrec_offsets_compare (struct data_reference *,
! struct data_reference *);
extern bool runtime_alias_check_p (ddr_p, struct loop *, bool);
extern int data_ref_compare_tree (tree, tree);
Index: gcc/tree-data-ref.c
===================================================================
*** gcc/tree-data-ref.c 2017-08-04 11:42:45.938134820 +0100
--- gcc/tree-data-ref.c 2017-08-16 14:34:56.611554296 +0100
*************** Software Foundation; either version 3, o
*** 86,91 ****
--- 86,92 ----
#include "expr.h"
#include "gimple-iterator.h"
#include "tree-ssa-loop-niter.h"
+ #include "tree-ssa-loop-ivopts.h"
#include "tree-ssa-loop.h"
#include "tree-ssa.h"
#include "cfgloop.h"
*************** split_constant_offset (tree exp, tree *v
*** 730,736 ****
*off = ssize_int (0);
STRIP_NOPS (exp);
! if (tree_is_chrec (exp)
|| get_gimple_rhs_class (TREE_CODE (exp)) == GIMPLE_TERNARY_RHS)
return;
--- 731,749 ----
*off = ssize_int (0);
STRIP_NOPS (exp);
! if (TREE_CODE (exp) == POLYNOMIAL_CHREC)
! {
! split_constant_offset (CHREC_LEFT (exp), &op0, &op1);
! if (op0 != CHREC_LEFT (exp))
! {
! *var = build3 (POLYNOMIAL_CHREC, type, CHREC_VAR (exp),
! op0, CHREC_RIGHT (exp));
! *off = op1;
! }
! return;
! }
!
! if (automatically_generated_chrec_p (exp)
|| get_gimple_rhs_class (TREE_CODE (exp)) == GIMPLE_TERNARY_RHS)
return;
*************** dr_analyze_innermost (innermost_loop_beh
*** 924,929 ****
--- 937,944 ----
drb->offset = fold_convert (ssizetype, offset_iv.base);
drb->init = init;
drb->step = step;
+ drb->chrec_offset = NULL_TREE;
+ drb->chrec_init = NULL_TREE;
drb->base_alignment = base_alignment;
drb->base_misalignment = base_misalignment & (base_alignment - 1);
drb->offset_alignment = highest_pow2_factor (offset_iv.base);
*************** runtime_alias_check_p (ddr_p ddr, struct
*** 1324,1334 ****
operator == (const dr_with_seg_len& d1,
const dr_with_seg_len& d2)
{
! return operand_equal_p (DR_BASE_ADDRESS (d1.dr),
DR_BASE_ADDRESS (d2.dr), 0)
! && data_ref_compare_tree (DR_OFFSET (d1.dr), DR_OFFSET (d2.dr)) == 0
! && data_ref_compare_tree (DR_INIT (d1.dr), DR_INIT (d2.dr)) == 0
! && data_ref_compare_tree (d1.seg_len, d2.seg_len) == 0;
}
/* Comparison function for sorting objects of dr_with_seg_len_pair_t
--- 1339,1350 ----
operator == (const dr_with_seg_len& d1,
const dr_with_seg_len& d2)
{
! return (operand_equal_p (DR_BASE_ADDRESS (d1.dr),
DR_BASE_ADDRESS (d2.dr), 0)
! && dr_chrec_offsets_equal_p (d1.dr, d2.dr)
! && data_ref_compare_tree (DR_CHREC_INIT (d1.dr),
! DR_CHREC_INIT (d2.dr)) == 0
! && data_ref_compare_tree (d1.seg_len, d2.seg_len) == 0);
}
/* Comparison function for sorting objects of dr_with_seg_len_pair_t
*************** comp_dr_with_seg_len_pair (const void *p
*** 1360,1376 ****
if ((comp_res = data_ref_compare_tree (DR_STEP (a2.dr),
DR_STEP (b2.dr))) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_OFFSET (a1.dr),
! DR_OFFSET (b1.dr))) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_INIT (a1.dr),
! DR_INIT (b1.dr))) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_OFFSET (a2.dr),
! DR_OFFSET (b2.dr))) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_INIT (a2.dr),
! DR_INIT (b2.dr))) != 0)
return comp_res;
return 0;
--- 1376,1390 ----
if ((comp_res = data_ref_compare_tree (DR_STEP (a2.dr),
DR_STEP (b2.dr))) != 0)
return comp_res;
! if ((comp_res = dr_chrec_offsets_compare (a1.dr, b1.dr)) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_CHREC_INIT (a1.dr),
! DR_CHREC_INIT (b1.dr))) != 0)
return comp_res;
! if ((comp_res = dr_chrec_offsets_compare (a2.dr, b2.dr)) != 0)
return comp_res;
! if ((comp_res = data_ref_compare_tree (DR_CHREC_INIT (a2.dr),
! DR_CHREC_INIT (b2.dr))) != 0)
return comp_res;
return 0;
*************** prune_runtime_alias_test_list (vec<dr_wi
*** 1455,1464 ****
if (!operand_equal_p (DR_BASE_ADDRESS (dr_a1->dr),
DR_BASE_ADDRESS (dr_a2->dr), 0)
! || !operand_equal_p (DR_OFFSET (dr_a1->dr),
! DR_OFFSET (dr_a2->dr), 0)
! || !tree_fits_shwi_p (DR_INIT (dr_a1->dr))
! || !tree_fits_shwi_p (DR_INIT (dr_a2->dr)))
continue;
/* Only merge const step data references. */
--- 1469,1477 ----
if (!operand_equal_p (DR_BASE_ADDRESS (dr_a1->dr),
DR_BASE_ADDRESS (dr_a2->dr), 0)
! || !dr_chrec_offsets_equal_p (dr_a1->dr, dr_a2->dr)
! || !tree_fits_shwi_p (DR_CHREC_INIT (dr_a1->dr))
! || !tree_fits_shwi_p (DR_CHREC_INIT (dr_a2->dr)))
continue;
/* Only merge const step data references. */
*************** prune_runtime_alias_test_list (vec<dr_wi
*** 1484,1494 ****
continue;
/* Make sure dr_a1 starts left of dr_a2. */
! if (tree_int_cst_lt (DR_INIT (dr_a2->dr), DR_INIT (dr_a1->dr)))
std::swap (*dr_a1, *dr_a2);
bool do_remove = false;
! wide_int diff = wi::sub (DR_INIT (dr_a2->dr), DR_INIT (dr_a1->dr));
wide_int min_seg_len_b;
tree new_seg_len;
--- 1497,1509 ----
continue;
/* Make sure dr_a1 starts left of dr_a2. */
! if (tree_int_cst_lt (DR_CHREC_INIT (dr_a2->dr),
! DR_CHREC_INIT (dr_a1->dr)))
std::swap (*dr_a1, *dr_a2);
bool do_remove = false;
! wide_int diff = wi::sub (DR_CHREC_INIT (dr_a2->dr),
! DR_CHREC_INIT (dr_a1->dr));
wide_int min_seg_len_b;
tree new_seg_len;
*************** create_runtime_alias_checks (struct loop
*** 1826,1871 ****
}
}
! /* Check if OFFSET1 and OFFSET2 (DR_OFFSETs of some data-refs) are identical
! expressions. */
! static bool
! dr_equal_offsets_p1 (tree offset1, tree offset2)
{
! bool res;
! STRIP_NOPS (offset1);
! STRIP_NOPS (offset2);
! if (offset1 == offset2)
! return true;
! if (TREE_CODE (offset1) != TREE_CODE (offset2)
! || (!BINARY_CLASS_P (offset1) && !UNARY_CLASS_P (offset1)))
! return false;
! res = dr_equal_offsets_p1 (TREE_OPERAND (offset1, 0),
! TREE_OPERAND (offset2, 0));
! if (!res || !BINARY_CLASS_P (offset1))
! return res;
! res = dr_equal_offsets_p1 (TREE_OPERAND (offset1, 1),
! TREE_OPERAND (offset2, 1));
! return res;
}
! /* Check if DRA and DRB have equal offsets. */
! bool
! dr_equal_offsets_p (struct data_reference *dra,
! struct data_reference *drb)
! {
! tree offset1, offset2;
! offset1 = DR_OFFSET (dra);
! offset2 = DR_OFFSET (drb);
! return dr_equal_offsets_p1 (offset1, offset2);
}
/* Returns true if FNA == FNB. */
--- 1841,1923 ----
}
}
! /* Analyze the scalar evolution of OFFSET in the innermost parent of
! LOOP for which it isn't invariant. */
!
! static tree
! analyze_offset_scev (struct loop *loop, tree offset)
{
! struct loop *inv_loop = outermost_invariant_loop_for_expr (loop, offset);
! if (inv_loop != NULL)
! {
! if (loop_depth (inv_loop) == 0)
! return offset;
! loop = loop_outer (inv_loop);
! }
! return analyze_scalar_evolution (loop, offset);
! }
! /* Analyze the scalar evolution of DRB->offset + DRB->init in a form
! that is valid for use in LOOP. Store the result DRB->chrec_offset
! and DRB->chrec_init. */
! static void
! compute_offset_chrecs (struct loop *loop, innermost_loop_behavior *drb)
! {
! tree scev = analyze_offset_scev (loop, drb->offset);
! if (TREE_CODE (scev) == POLYNOMIAL_CHREC)
! {
! tree init_term;
! split_constant_offset (scev, &drb->chrec_offset, &init_term);
! drb->chrec_init = fold_build2 (PLUS_EXPR, ssizetype, init_term,
! drb->init);
! }
! else
! {
! drb->chrec_offset = drb->offset;
! drb->chrec_init = drb->init;
! }
! }
! /* Analyze the scalar evolution of DR_OFFSET (DR) + DR_INIT (DR) wrt
! the loop that contains DR_STMT (DR), storing the result in
! DR_CHREC_OFFSET (DR) and DR_CHREC_INIT (DR). */
! static void
! compute_offset_chrecs (data_reference *dr)
! {
! return compute_offset_chrecs (loop_containing_stmt (DR_STMT (dr)),
! &DR_INNERMOST (dr));
! }
! /* Check if DRA and DRB have equal DR_CHREC_OFFSETs, computing them
! first if necessary. */
! bool
! dr_chrec_offsets_equal_p (struct data_reference *dra,
! struct data_reference *drb)
! {
! if (!DR_CHREC_OFFSET (dra))
! compute_offset_chrecs (dra);
! if (!DR_CHREC_OFFSET (drb))
! compute_offset_chrecs (drb);
! return eq_evolutions_p (DR_CHREC_OFFSET (dra), DR_CHREC_OFFSET (drb));
}
! /* Compare the DR_CHREC_OFFSETs of DRA and DRB for sorting purposes,
! computing them first if necessary. */
! int
! dr_chrec_offsets_compare (struct data_reference *dra,
! struct data_reference *drb)
! {
! if (!DR_CHREC_OFFSET (dra))
! compute_offset_chrecs (dra);
! if (!DR_CHREC_OFFSET (drb))
! compute_offset_chrecs (drb);
! return data_ref_compare_tree (DR_CHREC_OFFSET (dra), DR_CHREC_OFFSET (drb));
}
/* Returns true if FNA == FNB. */
Index: gcc/tree-loop-distribution.c
===================================================================
*** gcc/tree-loop-distribution.c 2017-08-16 08:50:32.415427038 +0100
--- gcc/tree-loop-distribution.c 2017-08-16 14:34:56.612554255 +0100
*************** compute_alias_check_pairs (struct loop *
*** 2204,2210 ****
DR_BASE_ADDRESS (dr_b));
if (comp_res == 0)
! comp_res = data_ref_compare_tree (DR_OFFSET (dr_a), DR_OFFSET (dr_b));
gcc_assert (comp_res != 0);
if (latch_dominated_by_data_ref (loop, dr_a))
--- 2204,2210 ----
DR_BASE_ADDRESS (dr_b));
if (comp_res == 0)
! comp_res = dr_chrec_offsets_compare (dr_a, dr_b);
gcc_assert (comp_res != 0);
if (latch_dominated_by_data_ref (loop, dr_a))
Index: gcc/tree-vect-data-refs.c
===================================================================
*** gcc/tree-vect-data-refs.c 2017-08-04 11:42:45.939105152 +0100
--- gcc/tree-vect-data-refs.c 2017-08-16 14:34:56.613554213 +0100
*************** vect_find_same_alignment_drs (struct dat
*** 2187,2199 ****
return;
if (!operand_equal_p (DR_BASE_ADDRESS (dra), DR_BASE_ADDRESS (drb), 0)
! || !operand_equal_p (DR_OFFSET (dra), DR_OFFSET (drb), 0)
|| !operand_equal_p (DR_STEP (dra), DR_STEP (drb), 0))
return;
/* Two references with distance zero have the same alignment. */
! offset_int diff = (wi::to_offset (DR_INIT (dra))
! - wi::to_offset (DR_INIT (drb)));
if (diff != 0)
{
/* Get the wider of the two alignments. */
--- 2187,2199 ----
return;
if (!operand_equal_p (DR_BASE_ADDRESS (dra), DR_BASE_ADDRESS (drb), 0)
! || !dr_chrec_offsets_equal_p (dra, drb)
|| !operand_equal_p (DR_STEP (dra), DR_STEP (drb), 0))
return;
/* Two references with distance zero have the same alignment. */
! offset_int diff = (wi::to_offset (DR_CHREC_INIT (dra))
! - wi::to_offset (DR_CHREC_INIT (drb)));
if (diff != 0)
{
/* Get the wider of the two alignments. */
*************** vect_analyze_group_access_1 (struct data
*** 2434,2440 ****
gimple *next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt));
struct data_reference *data_ref = dr;
unsigned int count = 1;
! tree prev_init = DR_INIT (data_ref);
gimple *prev = stmt;
HOST_WIDE_INT diff, gaps = 0;
--- 2434,2440 ----
gimple *next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt));
struct data_reference *data_ref = dr;
unsigned int count = 1;
! tree prev_init = DR_CHREC_INIT (data_ref);
gimple *prev = stmt;
HOST_WIDE_INT diff, gaps = 0;
*************** vect_analyze_group_access_1 (struct data
*** 2444,2452 ****
data-ref (supported only for loads), we vectorize only the first
stmt, and the rest get their vectorized loads from the first
one. */
! if (!tree_int_cst_compare (DR_INIT (data_ref),
! DR_INIT (STMT_VINFO_DATA_REF (
! vinfo_for_stmt (next)))))
{
if (DR_IS_WRITE (data_ref))
{
--- 2444,2452 ----
data-ref (supported only for loads), we vectorize only the first
stmt, and the rest get their vectorized loads from the first
one. */
! if (!tree_int_cst_compare (DR_CHREC_INIT (data_ref),
! DR_CHREC_INIT (STMT_VINFO_DATA_REF
! (vinfo_for_stmt (next)))))
{
if (DR_IS_WRITE (data_ref))
{
*************** vect_analyze_group_access_1 (struct data
*** 2476,2482 ****
/* Check that the distance between two accesses is equal to the type
size. Otherwise, we have gaps. */
! diff = (TREE_INT_CST_LOW (DR_INIT (data_ref))
- TREE_INT_CST_LOW (prev_init)) / type_size;
if (diff != 1)
{
--- 2476,2482 ----
/* Check that the distance between two accesses is equal to the type
size. Otherwise, we have gaps. */
! diff = (TREE_INT_CST_LOW (DR_CHREC_INIT (data_ref))
- TREE_INT_CST_LOW (prev_init)) / type_size;
if (diff != 1)
{
*************** vect_analyze_group_access_1 (struct data
*** 2499,2505 ****
gap in the access, GROUP_GAP is always 1. */
GROUP_GAP (vinfo_for_stmt (next)) = diff;
! prev_init = DR_INIT (data_ref);
next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next));
/* Count the number of data-refs in the chain. */
count++;
--- 2499,2505 ----
gap in the access, GROUP_GAP is always 1. */
GROUP_GAP (vinfo_for_stmt (next)) = diff;
! prev_init = DR_CHREC_INIT (data_ref);
next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next));
/* Count the number of data-refs in the chain. */
count++;
*************** dr_group_sort_cmp (const void *dra_, con
*** 2715,2727 ****
return cmp;
}
! /* And according to DR_OFFSET. */
! if (!dr_equal_offsets_p (dra, drb))
! {
! cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
! if (cmp != 0)
! return cmp;
! }
/* Put reads before writes. */
if (DR_IS_READ (dra) != DR_IS_READ (drb))
--- 2715,2724 ----
return cmp;
}
! /* And according to DR_CHREC_OFFSET. */
! cmp = dr_chrec_offsets_compare (dra, drb);
! if (cmp != 0)
! return cmp;
/* Put reads before writes. */
if (DR_IS_READ (dra) != DR_IS_READ (drb))
*************** dr_group_sort_cmp (const void *dra_, con
*** 2745,2752 ****
return cmp;
}
! /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */
! cmp = tree_int_cst_compare (DR_INIT (dra), DR_INIT (drb));
if (cmp == 0)
return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
return cmp;
--- 2742,2750 ----
return cmp;
}
! /* Then sort after DR_CHREC_INIT. In case of identical DRs sort after
! stmt UID. */
! cmp = tree_int_cst_compare (DR_CHREC_INIT (dra), DR_CHREC_INIT (drb));
if (cmp == 0)
return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
return cmp;
*************** vect_analyze_data_ref_accesses (vec_info
*** 2817,2823 ****
if (DR_IS_READ (dra) != DR_IS_READ (drb)
|| !operand_equal_p (DR_BASE_ADDRESS (dra),
DR_BASE_ADDRESS (drb), 0)
! || !dr_equal_offsets_p (dra, drb)
|| !gimple_assign_single_p (DR_STMT (dra))
|| !gimple_assign_single_p (DR_STMT (drb)))
break;
--- 2815,2821 ----
if (DR_IS_READ (dra) != DR_IS_READ (drb)
|| !operand_equal_p (DR_BASE_ADDRESS (dra),
DR_BASE_ADDRESS (drb), 0)
! || !dr_chrec_offsets_equal_p (dra, drb)
|| !gimple_assign_single_p (DR_STMT (dra))
|| !gimple_assign_single_p (DR_STMT (drb)))
break;
*************** vect_analyze_data_ref_accesses (vec_info
*** 2835,2841 ****
break;
/* Do not place the same access in the interleaving chain twice. */
! if (tree_int_cst_compare (DR_INIT (dra), DR_INIT (drb)) == 0)
break;
/* Check the types are compatible.
--- 2833,2840 ----
break;
/* Do not place the same access in the interleaving chain twice. */
! if (tree_int_cst_compare (DR_CHREC_INIT (dra),
! DR_CHREC_INIT (drb)) == 0)
break;
/* Check the types are compatible.
*************** vect_analyze_data_ref_accesses (vec_info
*** 2844,2852 ****
TREE_TYPE (DR_REF (drb))))
break;
! /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb). */
! HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
! HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
gcc_assert (init_a <= init_b);
/* If init_b == init_a + the size of the type * k, we have an
--- 2843,2852 ----
TREE_TYPE (DR_REF (drb))))
break;
! /* Sorting has ensured that
! DR_CHREC_INIT (dra) <= DR_CHREC_INIT (drb). */
! HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_CHREC_INIT (dra));
! HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_CHREC_INIT (drb));
gcc_assert (init_a <= init_b);
/* If init_b == init_a + the size of the type * k, we have an
*************** vect_analyze_data_ref_accesses (vec_info
*** 2859,2868 ****
/* If we have a store, the accesses are adjacent. This splits
groups into chunks we support (we don't support vectorization
of stores with gaps). */
if (!DR_IS_READ (dra)
! && (init_b - (HOST_WIDE_INT) TREE_INT_CST_LOW
! (DR_INIT (datarefs_copy[i-1]))
! != type_size_a))
break;
/* If the step (if not zero or non-constant) is greater than the
--- 2859,2868 ----
/* If we have a store, the accesses are adjacent. This splits
groups into chunks we support (we don't support vectorization
of stores with gaps). */
+ HOST_WIDE_INT prev_init
+ = TREE_INT_CST_LOW (DR_CHREC_INIT (datarefs_copy[i - 1]));
if (!DR_IS_READ (dra)
! && (init_b - prev_init) != type_size_a)
break;
/* If the step (if not zero or non-constant) is greater than the
*************** vect_vfa_segment_size (struct data_refer
*** 2974,2985 ****
vect_no_alias_p (struct data_reference *a, struct data_reference *b,
tree segment_length_a, tree segment_length_b)
{
! gcc_assert (TREE_CODE (DR_INIT (a)) == INTEGER_CST
! && TREE_CODE (DR_INIT (b)) == INTEGER_CST);
! if (tree_int_cst_equal (DR_INIT (a), DR_INIT (b)))
return false;
! tree seg_a_min = DR_INIT (a);
tree seg_a_max = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_a_min),
seg_a_min, segment_length_a);
/* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
--- 2974,2985 ----
vect_no_alias_p (struct data_reference *a, struct data_reference *b,
tree segment_length_a, tree segment_length_b)
{
! gcc_assert (TREE_CODE (DR_CHREC_INIT (a)) == INTEGER_CST
! && TREE_CODE (DR_CHREC_INIT (b)) == INTEGER_CST);
! if (tree_int_cst_equal (DR_CHREC_INIT (a), DR_CHREC_INIT (b)))
return false;
! tree seg_a_min = DR_CHREC_INIT (a);
tree seg_a_max = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_a_min),
seg_a_min, segment_length_a);
/* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
*************** vect_no_alias_p (struct data_reference *
*** 2990,2999 ****
tree unit_size = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (a)));
seg_a_min = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_a_max),
seg_a_max, unit_size);
! seg_a_max = fold_build2 (PLUS_EXPR, TREE_TYPE (DR_INIT (a)),
! DR_INIT (a), unit_size);
}
! tree seg_b_min = DR_INIT (b);
tree seg_b_max = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_b_min),
seg_b_min, segment_length_b);
if (tree_int_cst_compare (DR_STEP (b), size_zero_node) < 0)
--- 2990,2999 ----
tree unit_size = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (a)));
seg_a_min = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_a_max),
seg_a_max, unit_size);
! seg_a_max = fold_build2 (PLUS_EXPR, TREE_TYPE (DR_CHREC_INIT (a)),
! DR_CHREC_INIT (a), unit_size);
}
! tree seg_b_min = DR_CHREC_INIT (b);
tree seg_b_max = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_b_min),
seg_b_min, segment_length_b);
if (tree_int_cst_compare (DR_STEP (b), size_zero_node) < 0)
*************** vect_no_alias_p (struct data_reference *
*** 3001,3008 ****
tree unit_size = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (b)));
seg_b_min = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_b_max),
seg_b_max, unit_size);
! seg_b_max = fold_build2 (PLUS_EXPR, TREE_TYPE (DR_INIT (b)),
! DR_INIT (b), unit_size);
}
if (tree_int_cst_le (seg_a_max, seg_b_min)
--- 3001,3008 ----
tree unit_size = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (b)));
seg_b_min = fold_build2 (PLUS_EXPR, TREE_TYPE (seg_b_max),
seg_b_max, unit_size);
! seg_b_max = fold_build2 (PLUS_EXPR, TREE_TYPE (DR_CHREC_INIT (b)),
! DR_CHREC_INIT (b), unit_size);
}
if (tree_int_cst_le (seg_a_max, seg_b_min)
*************** vect_prune_runtime_alias_test_list (loop
*** 3148,3155 ****
comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (dr_a),
DR_BASE_ADDRESS (dr_b));
if (comp_res == 0)
! comp_res = data_ref_compare_tree (DR_OFFSET (dr_a),
! DR_OFFSET (dr_b));
/* Alias is known at compilation time. */
if (comp_res == 0
--- 3148,3154 ----
comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (dr_a),
DR_BASE_ADDRESS (dr_b));
if (comp_res == 0)
! comp_res = dr_chrec_offsets_compare (dr_a, dr_b);
/* Alias is known at compilation time. */
if (comp_res == 0
Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c 2017-08-03 10:40:55.650125801 +0100
--- gcc/tree-vect-stmts.c 2017-08-16 14:34:56.614554172 +0100
*************** vectorizable_load (gimple *stmt, gimple_
*** 7423,7430 ****
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr));
tree diff = fold_convert (sizetype,
size_binop (MINUS_EXPR,
! DR_INIT (first_dr),
! DR_INIT (ptrdr)));
dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
stmt, diff);
}
--- 7423,7430 ----
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr));
tree diff = fold_convert (sizetype,
size_binop (MINUS_EXPR,
! DR_CHREC_INIT (first_dr),
! DR_CHREC_INIT (ptrdr)));
dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
stmt, diff);
}
Index: gcc/testsuite/gcc.dg/vect/bb-slp-pr81635.c
===================================================================
*** /dev/null 2017-08-15 18:19:06.926776201 +0100
--- gcc/testsuite/gcc.dg/vect/bb-slp-pr81635.c 2017-08-16 14:34:56.610554337 +0100
***************
*** 0 ****
--- 1,32 ----
+ /* { dg-do compile } */
+ /* { dg-require-effective-target vect_unpack } */
+
+ double p[1000];
+ double q[1000];
+
+ void
+ f1 (void)
+ {
+ for (unsigned int i = 0; i < 1000; i += 4)
+ {
+ double a = q[i] + p[i];
+ double b = q[i + 1] + p[i + 1];
+ q[i] = a;
+ q[i + 1] = b;
+ }
+ }
+
+ void
+ f2 (void)
+ {
+ for (unsigned int i = 0; i < 500; i += 6)
+ for (unsigned int j = 0; j < 500; j += 4)
+ {
+ double a = q[j] + p[i];
+ double b = q[j + 1] + p[i + 1];
+ q[i] = a;
+ q[i + 1] = b;
+ }
+ }
+
+ /* { dg-final { scan-tree-dump-times "basic block vectorized" 2 "slp1" } } */