[mask conversion, patch 1/2] Add pattern for mask conversions
Richard Biener
richard.guenther@gmail.com
Tue Oct 27 13:35:00 GMT 2015
On Mon, Oct 19, 2015 at 2:22 PM, Ilya Enkovich <enkovich.gnu@gmail.com> wrote:
> Hi,
>
> This patch adds a vectorization pattern which detects cases where mask conversion is needed and adds it. It is done for all statements which may consume mask. Some additional changes were made to support MASK_LOAD with pattern and allow scalar mode for vectype of pattern stmt. It is applied on top of all other boolean vector series. Does it look OK?
Ick - more patterns. I was really hoping we could get away without
patterns for bools :/
Ok.
Thanks,
Richard.
> Thanks,
> Ilya
> --
> gcc/
>
> 2015-10-19 Ilya Enkovich <enkovich.gnu@gmail.com>
>
> * optabs.c (expand_binop_directly): Allow scalar mode for
> vec_pack_trunc_optab.
> * tree-vect-loop.c (vect_determine_vectorization_factor): Skip
> boolean vector producers from pattern sequence when computing VF.
> * tree-vect-patterns.c (vect_vect_recog_func_ptrs) Add
> vect_recog_mask_conversion_pattern.
> (search_type_for_mask): Choose the smallest
> type if different size types are mixed.
> (build_mask_conversion): New.
> (vect_recog_mask_conversion_pattern): New.
> (vect_pattern_recog_1): Allow scalar mode for boolean vectype.
> * tree-vect-stmts.c (vectorizable_mask_load_store): Support masked
> load with pattern.
> (vectorizable_conversion): Support boolean vectors.
> (free_stmt_vec_info): Allow patterns for statements with no lhs.
> * tree-vectorizer.h (NUM_PATTERNS): Increase to 14.
>
>
> diff --git a/gcc/optabs.c b/gcc/optabs.c
> index 83f4be3..8d61d33 100644
> --- a/gcc/optabs.c
> +++ b/gcc/optabs.c
> @@ -1055,7 +1055,8 @@ expand_binop_directly (machine_mode mode, optab binoptab,
> /* The mode of the result is different then the mode of the
> arguments. */
> tmp_mode = insn_data[(int) icode].operand[0].mode;
> - if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
> + if (VECTOR_MODE_P (mode)
> + && GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
> {
> delete_insns_since (last);
> return NULL_RTX;
> diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
> index 14804b3..e388533 100644
> --- a/gcc/tree-vect-loop.c
> +++ b/gcc/tree-vect-loop.c
> @@ -497,6 +497,17 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
> }
> }
>
> + /* Boolean vectors don't affect VF. */
> + if (VECTOR_BOOLEAN_TYPE_P (vectype))
> + {
> + if (!analyze_pattern_stmt && gsi_end_p (pattern_def_si))
> + {
> + pattern_def_seq = NULL;
> + gsi_next (&si);
> + }
> + continue;
> + }
> +
> /* The vectorization factor is according to the smallest
> scalar type (or the largest vector size, but we only
> support one vector size per loop). */
> diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
> index a737129..34b1ea6 100644
> --- a/gcc/tree-vect-patterns.c
> +++ b/gcc/tree-vect-patterns.c
> @@ -76,6 +76,7 @@ static gimple *vect_recog_mult_pattern (vec<gimple *> *,
> static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *,
> tree *, tree *);
> static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *);
> +static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *);
> static vect_recog_func_ptr vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
> vect_recog_widen_mult_pattern,
> vect_recog_widen_sum_pattern,
> @@ -89,7 +90,8 @@ static vect_recog_func_ptr vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
> vect_recog_divmod_pattern,
> vect_recog_mult_pattern,
> vect_recog_mixed_size_cond_pattern,
> - vect_recog_bool_pattern};
> + vect_recog_bool_pattern,
> + vect_recog_mask_conversion_pattern};
>
> static inline void
> append_pattern_def_seq (stmt_vec_info stmt_info, gimple *stmt)
> @@ -3180,7 +3182,7 @@ search_type_for_mask (tree var, vec_info *vinfo)
> enum vect_def_type dt;
> tree rhs1;
> enum tree_code rhs_code;
> - tree res = NULL;
> + tree res = NULL, res2;
>
> if (TREE_CODE (var) != SSA_NAME)
> return NULL;
> @@ -3213,13 +3215,26 @@ search_type_for_mask (tree var, vec_info *vinfo)
> case BIT_AND_EXPR:
> case BIT_IOR_EXPR:
> case BIT_XOR_EXPR:
> - if (!(res = search_type_for_mask (rhs1, vinfo)))
> - res = search_type_for_mask (gimple_assign_rhs2 (def_stmt), vinfo);
> + res = search_type_for_mask (rhs1, vinfo);
> + res2 = search_type_for_mask (gimple_assign_rhs2 (def_stmt), vinfo);
> + if (!res || (res2 && TYPE_PRECISION (res) > TYPE_PRECISION (res2)))
> + res = res2;
> break;
>
> default:
> if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
> {
> + tree comp_vectype, mask_type;
> +
> + comp_vectype = get_vectype_for_scalar_type (TREE_TYPE (rhs1));
> + if (comp_vectype == NULL_TREE)
> + return NULL;
> +
> + mask_type = get_mask_type_for_scalar_type (TREE_TYPE (rhs1));
> + if (!mask_type
> + || !expand_vec_cmp_expr_p (comp_vectype, mask_type))
> + return NULL;
> +
> if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE
> || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
> {
> @@ -3481,6 +3496,255 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
> }
>
>
> +/* A helper for vect_recog_mask_conversion_pattern. Build
> + conversion of MASK to a type suitable for masking VECTYPE.
> + Built statement gets required vectype and is appended to
> + a pattern sequence of STMT_VINFO.
> +
> + Return converted mask. */
> +
> +static tree
> +build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo,
> + vec_info *vinfo)
> +{
> + gimple *stmt;
> + tree masktype, tmp;
> + stmt_vec_info new_stmt_info;
> +
> + masktype = build_same_sized_truth_vector_type (vectype);
> + tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
> + stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
> + new_stmt_info = new_stmt_vec_info (stmt, vinfo);
> + set_vinfo_for_stmt (stmt, new_stmt_info);
> + STMT_VINFO_VECTYPE (new_stmt_info) = masktype;
> + append_pattern_def_seq (stmt_vinfo, stmt);
> +
> + return tmp;
> +}
> +
> +
> +/* Function vect_recog_mask_conversion_pattern
> +
> + Try to find statements which require boolean type
> + converison. Additional conversion statements are
> + added to handle such cases. For example:
> +
> + bool m_1, m_2, m_3;
> + int i_4, i_5;
> + double d_6, d_7;
> + char c_1, c_2, c_3;
> +
> + S1 m_1 = i_4 > i_5;
> + S2 m_2 = d_6 < d_7;
> + S3 m_3 = m_1 & m_2;
> + S4 c_1 = m_3 ? c_2 : c_3;
> +
> + Will be transformed into:
> +
> + S1 m_1 = i_4 > i_5;
> + S2 m_2 = d_6 < d_7;
> + S3'' m_2' = (_Bool[bitsize=32])m_2
> + S3' m_3' = m_1 & m_2';
> + S4'' m_3'' = (_Bool[bitsize=8])m_3'
> + S4' c_1' = m_3'' ? c_2 : c_3; */
> +
> +static gimple *
> +vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
> + tree *type_out)
> +{
> + gimple *last_stmt = stmts->pop ();
> + enum tree_code rhs_code;
> + tree lhs, rhs1, rhs2, tmp, rhs1_type, rhs2_type, vectype1, vectype2;
> + stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
> + stmt_vec_info pattern_stmt_info;
> + vec_info *vinfo = stmt_vinfo->vinfo;
> + gimple *pattern_stmt;
> +
> + /* Check for MASK_LOAD ans MASK_STORE calls requiring mask conversion. */
> + if (is_gimple_call (last_stmt)
> + && gimple_call_internal_p (last_stmt)
> + && (gimple_call_internal_fn (last_stmt) == IFN_MASK_STORE
> + || gimple_call_internal_fn (last_stmt) == IFN_MASK_LOAD))
> + {
> + bool load = (gimple_call_internal_fn (last_stmt) == IFN_MASK_LOAD);
> +
> + if (load)
> + {
> + lhs = gimple_call_lhs (last_stmt);
> + vectype1 = get_vectype_for_scalar_type (TREE_TYPE (lhs));
> + }
> + else
> + {
> + rhs2 = gimple_call_arg (last_stmt, 3);
> + vectype1 = get_vectype_for_scalar_type (TREE_TYPE (rhs2));
> + }
> +
> + rhs1 = gimple_call_arg (last_stmt, 2);
> + rhs1_type = search_type_for_mask (rhs1, vinfo);
> + if (!rhs1_type)
> + return NULL;
> + vectype2 = get_mask_type_for_scalar_type (rhs1_type);
> +
> + if (!vectype1 || !vectype2
> + || TYPE_VECTOR_SUBPARTS (vectype1) == TYPE_VECTOR_SUBPARTS (vectype2))
> + return NULL;
> +
> + tmp = build_mask_conversion (rhs1, vectype1, stmt_vinfo, vinfo);
> +
> + if (load)
> + {
> + lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
> + pattern_stmt
> + = gimple_build_call_internal (IFN_MASK_LOAD, 3,
> + gimple_call_arg (last_stmt, 0),
> + gimple_call_arg (last_stmt, 1),
> + tmp);
> + gimple_call_set_lhs (pattern_stmt, lhs);
> + }
> + else
> + pattern_stmt
> + = gimple_build_call_internal (IFN_MASK_STORE, 4,
> + gimple_call_arg (last_stmt, 0),
> + gimple_call_arg (last_stmt, 1),
> + tmp,
> + gimple_call_arg (last_stmt, 3));
> +
> +
> + pattern_stmt_info = new_stmt_vec_info (pattern_stmt, vinfo);
> + set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
> + STMT_VINFO_DATA_REF (pattern_stmt_info)
> + = STMT_VINFO_DATA_REF (stmt_vinfo);
> + STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info)
> + = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo);
> + STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo);
> + STMT_VINFO_DR_OFFSET (pattern_stmt_info)
> + = STMT_VINFO_DR_OFFSET (stmt_vinfo);
> + STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo);
> + STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info)
> + = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo);
> + DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt;
> +
> + *type_out = vectype1;
> + *type_in = vectype1;
> + stmts->safe_push (last_stmt);
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "vect_recog_mask_conversion_pattern: detected:\n");
> +
> + return pattern_stmt;
> + }
> +
> + if (!is_gimple_assign (last_stmt))
> + return NULL;
> +
> + lhs = gimple_assign_lhs (last_stmt);
> + rhs1 = gimple_assign_rhs1 (last_stmt);
> + rhs_code = gimple_assign_rhs_code (last_stmt);
> +
> + /* Check for cond expression requiring mask conversion. */
> + if (rhs_code == COND_EXPR)
> + {
> + /* vect_recog_mixed_size_cond_pattern could apply.
> + Do nothing then. */
> + if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
> + return NULL;
> +
> + vectype1 = get_vectype_for_scalar_type (TREE_TYPE (lhs));
> +
> + if (TREE_CODE (rhs1) == SSA_NAME)
> + {
> + rhs1_type = search_type_for_mask (rhs1, vinfo);
> + if (!rhs1_type)
> + return NULL;
> + }
> + else
> + rhs1_type = TREE_TYPE (TREE_OPERAND (rhs1, 0));
> +
> + vectype2 = get_mask_type_for_scalar_type (rhs1_type);
> +
> + if (!vectype1 || !vectype2
> + || TYPE_VECTOR_SUBPARTS (vectype1) == TYPE_VECTOR_SUBPARTS (vectype2))
> + return NULL;
> +
> + /* If rhs1 is a comparison we need to move it into a
> + separate statement. */
> + if (TREE_CODE (rhs1) != SSA_NAME)
> + {
> + tmp = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
> + pattern_stmt = gimple_build_assign (tmp, rhs1);
> + rhs1 = tmp;
> +
> + pattern_stmt_info = new_stmt_vec_info (pattern_stmt, vinfo);
> + set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
> + STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype2;
> + append_pattern_def_seq (stmt_vinfo, pattern_stmt);
> + }
> +
> + tmp = build_mask_conversion (rhs1, vectype1, stmt_vinfo, vinfo);
> +
> + lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
> + pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
> + gimple_assign_rhs2 (last_stmt),
> + gimple_assign_rhs3 (last_stmt));
> +
> + *type_out = vectype1;
> + *type_in = vectype1;
> + stmts->safe_push (last_stmt);
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "vect_recog_mask_conversion_pattern: detected:\n");
> +
> + return pattern_stmt;
> + }
> +
> + /* Now check for binary boolean operations requiring conversion for
> + one of operands. */
> + if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE)
> + return NULL;
> +
> + if (rhs_code != BIT_IOR_EXPR
> + && rhs_code != BIT_XOR_EXPR
> + && rhs_code != BIT_AND_EXPR)
> + return NULL;
> +
> + rhs2 = gimple_assign_rhs2 (last_stmt);
> +
> + rhs1_type = search_type_for_mask (rhs1, vinfo);
> + rhs2_type = search_type_for_mask (rhs2, vinfo);
> +
> + if (!rhs1_type || !rhs2_type
> + || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
> + return NULL;
> +
> + if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
> + {
> + vectype1 = get_mask_type_for_scalar_type (rhs1_type);
> + if (!vectype1)
> + return NULL;
> + rhs2 = build_mask_conversion (rhs2, vectype1, stmt_vinfo, vinfo);
> + }
> + else
> + {
> + vectype1 = get_mask_type_for_scalar_type (rhs2_type);
> + if (!vectype1)
> + return NULL;
> + rhs1 = build_mask_conversion (rhs1, vectype1, stmt_vinfo, vinfo);
> + }
> +
> + lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
> + pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
> +
> + *type_out = vectype1;
> + *type_in = vectype1;
> + stmts->safe_push (last_stmt);
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "vect_recog_mask_conversion_pattern: detected:\n");
> +
> + return pattern_stmt;
> +}
> +
> +
> /* Mark statements that are involved in a pattern. */
>
> static inline void
> @@ -3576,7 +3840,8 @@ vect_pattern_recog_1 (vect_recog_func_ptr vect_recog_func,
> stmt_info = vinfo_for_stmt (stmt);
> loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
>
> - if (VECTOR_MODE_P (TYPE_MODE (type_in)))
> + if (VECTOR_BOOLEAN_TYPE_P (type_in)
> + || VECTOR_MODE_P (TYPE_MODE (type_in)))
> {
> /* No need to check target support (already checked by the pattern
> recognition function). */
> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
> index a600de6..9925858 100644
> --- a/gcc/tree-vect-stmts.c
> +++ b/gcc/tree-vect-stmts.c
> @@ -2004,6 +2004,11 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>
> /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
> from the IL. */
> + if (STMT_VINFO_RELATED_STMT (stmt_info))
> + {
> + stmt = STMT_VINFO_RELATED_STMT (stmt_info);
> + stmt_info = vinfo_for_stmt (stmt);
> + }
> tree lhs = gimple_call_lhs (stmt);
> new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
> set_vinfo_for_stmt (new_stmt, stmt_info);
> @@ -2122,6 +2127,11 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
> {
> /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
> from the IL. */
> + if (STMT_VINFO_RELATED_STMT (stmt_info))
> + {
> + stmt = STMT_VINFO_RELATED_STMT (stmt_info);
> + stmt_info = vinfo_for_stmt (stmt);
> + }
> tree lhs = gimple_call_lhs (stmt);
> new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
> set_vinfo_for_stmt (new_stmt, stmt_info);
> @@ -3592,12 +3602,13 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
> && SCALAR_FLOAT_TYPE_P (rhs_type))))
> return false;
>
> - if ((INTEGRAL_TYPE_P (lhs_type)
> - && (TYPE_PRECISION (lhs_type)
> - != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
> - || (INTEGRAL_TYPE_P (rhs_type)
> - && (TYPE_PRECISION (rhs_type)
> - != GET_MODE_PRECISION (TYPE_MODE (rhs_type)))))
> + if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
> + && ((INTEGRAL_TYPE_P (lhs_type)
> + && (TYPE_PRECISION (lhs_type)
> + != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
> + || (INTEGRAL_TYPE_P (rhs_type)
> + && (TYPE_PRECISION (rhs_type)
> + != GET_MODE_PRECISION (TYPE_MODE (rhs_type))))))
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -3655,6 +3666,21 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
> return false;
> }
>
> + if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
> + && !VECTOR_BOOLEAN_TYPE_P (vectype_in))
> + {
> + if (dump_enabled_p ())
> + {
> + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't convert between boolean and non "
> + "boolean vectors");
> + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
> + dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
> + }
> +
> + return false;
> + }
> +
> nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
> nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
> if (nunits_in < nunits_out)
> @@ -8230,7 +8256,7 @@ free_stmt_vec_info (gimple *stmt)
> gimple *patt_stmt = STMT_VINFO_STMT (patt_info);
> gimple_set_bb (patt_stmt, NULL);
> tree lhs = gimple_get_lhs (patt_stmt);
> - if (TREE_CODE (lhs) == SSA_NAME)
> + if (lhs && TREE_CODE (lhs) == SSA_NAME)
> release_ssa_name (lhs);
> if (seq)
> {
> @@ -8240,7 +8266,7 @@ free_stmt_vec_info (gimple *stmt)
> gimple *seq_stmt = gsi_stmt (si);
> gimple_set_bb (seq_stmt, NULL);
> lhs = gimple_get_lhs (seq_stmt);
> - if (TREE_CODE (lhs) == SSA_NAME)
> + if (lhs && TREE_CODE (lhs) == SSA_NAME)
> release_ssa_name (lhs);
> free_stmt_vec_info (seq_stmt);
> }
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index 1d5335f..904d491 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -1074,7 +1074,7 @@ extern void vect_slp_transform_bb (basic_block);
> Additional pattern recognition functions can (and will) be added
> in the future. */
> typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *);
> -#define NUM_PATTERNS 13
> +#define NUM_PATTERNS 14
> void vect_pattern_recog (vec_info *);
>
> /* In tree-vectorizer.c. */
More information about the Gcc-patches
mailing list