[gcc r14-4781] Rewrite more refs for epilogue vectorization

[Richard Biener]

https://gcc.gnu.org/g:e489464acf5541e6fdb4007c49266193cc93d16e

commit r14-4781-ge489464acf5541e6fdb4007c49266193cc93d16e
Author: Richard Biener <rguenther@suse.de>
Date:   Fri Oct 20 10:25:31 2023 +0200

    Rewrite more refs for epilogue vectorization
    
    The following makes sure to rewrite all gather/scatter detected by
    dataref analysis plus stmts classified as VMAT_GATHER_SCATTER.  Maybe
    we need to rewrite all refs, the following covers the cases I've
    run into now.
    
            * tree-vect-loop.cc (update_epilogue_loop_vinfo): Rewrite
            both STMT_VINFO_GATHER_SCATTER_P and VMAT_GATHER_SCATTER
            stmt refs.

Diff:
---
 gcc/tree-vect-loop.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 8877ebde2466..4a8b0a18800e 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -11361,8 +11361,12 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
       /* Data references for gather loads and scatter stores do not use the
 	 updated offset we set using ADVANCE.  Instead we have to make sure the
 	 reference in the data references point to the corresponding copy of
-	 the original in the epilogue.  */
-      if (STMT_VINFO_GATHER_SCATTER_P (vect_stmt_to_vectorize (stmt_vinfo)))
+	 the original in the epilogue.  Make sure to update both
+	 gather/scatters recognized by dataref analysis and also other
+	 refs that get_load_store_type classified as VMAT_GATHER_SCATTER.  */
+      auto vstmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
+      if (STMT_VINFO_MEMORY_ACCESS_TYPE (vstmt_vinfo) == VMAT_GATHER_SCATTER
+	  || STMT_VINFO_GATHER_SCATTER_P (vstmt_vinfo))
 	{
 	  DR_REF (dr)
 	    = simplify_replace_tree (DR_REF (dr), NULL_TREE, NULL_TREE,
@@ -11371,9 +11375,6 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 	    = simplify_replace_tree (DR_BASE_ADDRESS (dr), NULL_TREE, NULL_TREE,
 				     &find_in_mapping, &mapping);
 	}
-      else
-	gcc_assert (STMT_VINFO_MEMORY_ACCESS_TYPE (vect_stmt_to_vectorize (stmt_vinfo))
-		    != VMAT_GATHER_SCATTER);
       DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo);
       stmt_vinfo->dr_aux.stmt = stmt_vinfo;
       /* The vector size of the epilogue is smaller than that of the main loop