&& LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
- vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo),
- ncopies * vec_num, vectype, NULL);
+ {
+ if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
+ vectype, OPTIMIZE_FOR_SPEED))
+ vect_record_loop_len (loop_vinfo,
+ &LOOP_VINFO_LENS (loop_vinfo),
+ ncopies * vec_num, vectype, 1);
+ else
+ vect_record_loop_mask (loop_vinfo,
+ &LOOP_VINFO_MASKS (loop_vinfo),
+ ncopies * vec_num, vectype, NULL);
+ }
/* Extra inactive lanes should be safe for vect_nested_cycle. */
else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle)
{
mask to the condition, or to its inverse. */
vec_loop_masks *masks = NULL;
- if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ vec_loop_lens *lens = NULL;
+ if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+ {
+ if (reduction_type == EXTRACT_LAST_REDUCTION)
+ lens = &LOOP_VINFO_LENS (loop_vinfo);
+ }
+ else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
masks = &LOOP_VINFO_MASKS (loop_vinfo);
/* Force vec_compare to be an SSA_NAME rather than a comparison,
in cases where that's necessary. */
- if (masks || reduction_type == EXTRACT_LAST_REDUCTION)
+ tree len = NULL_TREE, bias = NULL_TREE;
+ if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
{
if (!is_gimple_val (vec_compare))
{
vec_compare = vec_compare_name;
}
+ if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
+ vectype, OPTIMIZE_FOR_SPEED))
+ {
+ if (lens)
+ {
+ len = vect_get_loop_len (loop_vinfo, gsi, lens,
+ vec_num * ncopies, vectype, i, 1);
+ signed char biasval
+ = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ bias = build_int_cst (intQI_type_node, biasval);
+ }
+ else
+ {
+ len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
+ bias = build_int_cst (intQI_type_node, 0);
+ }
+ }
if (masks)
{
tree loop_mask
{
gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
tree lhs = gimple_get_lhs (old_stmt);
- new_stmt = gimple_build_call_internal
- (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
- vec_then_clause);
+ if (len)
+ new_stmt = gimple_build_call_internal
+ (IFN_LEN_FOLD_EXTRACT_LAST, 5, else_clause, vec_compare,
+ vec_then_clause, len, bias);
+ else
+ new_stmt = gimple_build_call_internal
+ (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
+ vec_then_clause);
gimple_call_set_lhs (new_stmt, lhs);
SSA_NAME_DEF_STMT (lhs) = new_stmt;
if (old_stmt == gsi_stmt (*gsi))