2013-11-13 Jakub Jelinek * tree-vect-data-refs.c (vect_analyze_data_refs): Inline by hand find_data_references_in_loop and find_data_references_in_bb, if find_data_references_in_stmt fails, still allow calls to MASK_LOAD and MASK_STORE internal calls. --- gcc/tree-vect-data-refs.c.jj 2013-11-13 19:19:07.433383289 +0100 +++ gcc/tree-vect-data-refs.c 2013-11-13 20:27:11.209181089 +0100 @@ -3191,10 +3191,11 @@ vect_analyze_data_refs (loop_vec_info lo if (loop_vinfo) { + basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); + loop = LOOP_VINFO_LOOP (loop_vinfo); - if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)) - || find_data_references_in_loop - (loop, &LOOP_VINFO_DATAREFS (loop_vinfo))) + datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); + if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -3203,7 +3204,62 @@ vect_analyze_data_refs (loop_vec_info lo return false; } - datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); + for (i = 0; i < loop->num_nodes; i++) + { + gimple_stmt_iterator gsi; + + for (gsi = gsi_start_bb (bbs[i]); !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple stmt = gsi_stmt (gsi); + if (!find_data_references_in_stmt (loop, stmt, &datarefs)) + { + bool is_read = false; + tree ref; + data_reference_p drt; + + if (is_gimple_call (stmt) && gimple_call_internal_p (stmt)) + switch (gimple_call_internal_fn (stmt)) + { + case IFN_GOMP_SIMD_LANE: + { + struct loop *cloop = loop_containing_stmt (stmt); + tree uid = gimple_call_arg (stmt, 0); + gcc_assert (TREE_CODE (uid) == SSA_NAME); + if (cloop && cloop->simduid == SSA_NAME_VAR (uid)) + continue; + break; + } + case IFN_MASK_LOAD: + is_read = true; + /* FALLTHRU */ + case IFN_MASK_STORE: + ref = build2 (MEM_REF, + is_read + ? TREE_TYPE (gimple_call_lhs (stmt)) + : TREE_TYPE (gimple_call_arg (stmt, 3)), + gimple_call_arg (stmt, 0), + gimple_call_arg (stmt, 1)); + drt = create_data_ref (loop, + loop_containing_stmt (stmt), + ref, stmt, is_read); + gcc_assert (drt); + datarefs.safe_push (drt); + continue; + default: + break; + } + LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: loop contains function " + "calls or data references that cannot " + "be analyzed\n"); + return false; + } + } + } + + LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs; } else { --- gcc/tree-data-ref.c.jj 2013-11-13 19:19:07.000000000 +0100 +++ gcc/tree-data-ref.c 2013-11-13 20:28:16.753878385 +0100 @@ -4313,8 +4313,8 @@ compute_all_dependences (vecloop_father; - tree uid = gimple_call_arg (stmt, 0); - gcc_assert (TREE_CODE (uid) == SSA_NAME); - if (loop == NULL - || loop->simduid != SSA_NAME_VAR (uid)) - clobbers_memory = true; - break; - } - case IFN_MASK_LOAD: - case IFN_MASK_STORE: - break; - default: + if (gimple_call_internal_p (stmt) + && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE) + { + struct loop *loop = gimple_bb (stmt)->loop_father; + tree uid = gimple_call_arg (stmt, 0); + gcc_assert (TREE_CODE (uid) == SSA_NAME); + if (loop == NULL + || loop->simduid != SSA_NAME_VAR (uid)) clobbers_memory = true; - break; - } + } else clobbers_memory = true; } @@ -4372,15 +4362,15 @@ get_references_in_stmt (gimple stmt, vec if (stmt_code == GIMPLE_ASSIGN) { tree base; - op0 = gimple_assign_lhs (stmt); - op1 = gimple_assign_rhs1 (stmt); + op0 = gimple_assign_lhs_ptr (stmt); + op1 = gimple_assign_rhs1_ptr (stmt); - if (DECL_P (op1) - || (REFERENCE_CLASS_P (op1) - && (base = get_base_address (op1)) + if (DECL_P (*op1) + || (REFERENCE_CLASS_P (*op1) + && (base = get_base_address (*op1)) && TREE_CODE (base) != SSA_NAME)) { - ref.ref = op1; + ref.pos = op1; ref.is_read = true; references->safe_push (ref); } @@ -4389,35 +4379,16 @@ get_references_in_stmt (gimple stmt, vec { unsigned i, n; - ref.is_read = false; - if (gimple_call_internal_p (stmt)) - switch (gimple_call_internal_fn (stmt)) - { - case IFN_MASK_LOAD: - ref.is_read = true; - case IFN_MASK_STORE: - ref.ref = build2 (MEM_REF, - ref.is_read - ? TREE_TYPE (gimple_call_lhs (stmt)) - : TREE_TYPE (gimple_call_arg (stmt, 3)), - gimple_call_arg (stmt, 0), - gimple_call_arg (stmt, 1)); - references->safe_push (ref); - return false; - default: - break; - } - - op0 = gimple_call_lhs (stmt); + op0 = gimple_call_lhs_ptr (stmt); n = gimple_call_num_args (stmt); for (i = 0; i < n; i++) { - op1 = gimple_call_arg (stmt, i); + op1 = gimple_call_arg_ptr (stmt, i); - if (DECL_P (op1) - || (REFERENCE_CLASS_P (op1) && get_base_address (op1))) + if (DECL_P (*op1) + || (REFERENCE_CLASS_P (*op1) && get_base_address (*op1))) { - ref.ref = op1; + ref.pos = op1; ref.is_read = true; references->safe_push (ref); } @@ -4426,11 +4397,11 @@ get_references_in_stmt (gimple stmt, vec else return clobbers_memory; - if (op0 - && (DECL_P (op0) - || (REFERENCE_CLASS_P (op0) && get_base_address (op0)))) + if (*op0 + && (DECL_P (*op0) + || (REFERENCE_CLASS_P (*op0) && get_base_address (*op0)))) { - ref.ref = op0; + ref.pos = op0; ref.is_read = false; references->safe_push (ref); } @@ -4457,7 +4428,7 @@ find_data_references_in_stmt (struct loo FOR_EACH_VEC_ELT (references, i, ref) { dr = create_data_ref (nest, loop_containing_stmt (stmt), - ref->ref, stmt, ref->is_read); + *ref->pos, stmt, ref->is_read); gcc_assert (dr != NULL); datarefs->safe_push (dr); } @@ -4486,7 +4457,7 @@ graphite_find_data_references_in_stmt (l FOR_EACH_VEC_ELT (references, i, ref) { - dr = create_data_ref (nest, loop, ref->ref, stmt, ref->is_read); + dr = create_data_ref (nest, loop, *ref->pos, stmt, ref->is_read); gcc_assert (dr != NULL); datarefs->safe_push (dr); }