mirror/gcc
2
0
Fork 0
mirror of git://gcc.gnu.org/git/gcc.git synced 2025-03-20 05:50:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

add vec_info * parameters where needed

Browse Source 
Soonish we'll get SLP nodes which have no corresponding scalar
stmt and thus not stmt_vec_info and thus no way to get back to
the associated vec_info.  This patch makes the vec_info available
as part of the APIs instead of putting in that back-pointer into
the leaf data structures.

2020-05-05  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.h (_stmt_vec_info::vinfo): Remove.
	(STMT_VINFO_LOOP_VINFO): Likewise.
	(STMT_VINFO_BB_VINFO): Likewise.
	* tree-vect-data-refs.c: Adjust for the above, adding vec_info *
	parameters and adjusting calls.
	* tree-vect-loop-manip.c: Likewise.
	* tree-vect-loop.c: Likewise.
	* tree-vect-patterns.c: Likewise.
	* tree-vect-slp.c: Likewise.
	* tree-vect-stmts.c: Likewise.
	* tree-vectorizer.c: Likewise.

	* target.def (add_stmt_cost): Add vec_info * parameter.
	* target.h (stmt_in_inner_loop_p): Likewise.
	* targhooks.c (default_add_stmt_cost): Adjust.
	* doc/tm.texi: Re-generate.

	* config/aarch64/aarch64.c (aarch64_extending_load_p): Add
	vec_info * parameter and adjust.
	(aarch64_sve_adjust_stmt_cost): Likewise.
	(aarch64_add_stmt_cost): Likewise.
	* config/arm/arm.c (arm_add_stmt_cost): Likewise.
	* config/i386/i386.c (ix86_add_stmt_cost): Likewise.
	* config/rs6000/rs6000.c (rs6000_add_stmt_cost): Likewise.
This commit is contained in:
Richard Biener 2020-03-16 11:47:00 +01:00
parent 228646a64f
commit 308bc49688
16 changed files with 1169 additions and 1017 deletions

18
gcc/config/aarch64/aarch64.c

@ -13639,7 +13639,7 @@ aarch64_advsimd_ldp_stp_p (enum vect_cost_for_stmt kind,
/* Return true if STMT_INFO extends the result of a load. */
static bool
aarch64_extending_load_p (stmt_vec_info stmt_info)
aarch64_extending_load_p (class vec_info *vinfo, stmt_vec_info stmt_info)
{
gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
@ -13653,7 +13653,7 @@ aarch64_extending_load_p (stmt_vec_info stmt_info)
|| TYPE_PRECISION (lhs_type) <= TYPE_PRECISION (rhs_type))
return false;
stmt_vec_info def_stmt_info = stmt_info->vinfo->lookup_def (rhs);
stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
return (def_stmt_info
&& STMT_VINFO_DATA_REF (def_stmt_info)
&& DR_IS_READ (STMT_VINFO_DATA_REF (def_stmt_info)));
@ -13679,7 +13679,7 @@ aarch64_integer_truncation_p (stmt_vec_info stmt_info)
operate on vector type VECTYPE. Adjust the cost as necessary for SVE
targets. */
static unsigned int
aarch64_sve_adjust_stmt_cost (vect_cost_for_stmt kind,
aarch64_sve_adjust_stmt_cost (class vec_info *vinfo, vect_cost_for_stmt kind,
stmt_vec_info stmt_info, tree vectype,
unsigned int stmt_cost)
{
@ -13691,7 +13691,7 @@ aarch64_sve_adjust_stmt_cost (vect_cost_for_stmt kind,
on the fly. Optimistically assume that a load followed by an extension
will fold to this form during combine, and that the extension therefore
comes for free. */
if (kind == vector_stmt && aarch64_extending_load_p (stmt_info))
if (kind == vector_stmt && aarch64_extending_load_p (vinfo, stmt_info))
stmt_cost = 0;
/* For similar reasons, vector_stmt integer truncations are a no-op,
@ -13744,7 +13744,8 @@ aarch64_sve_adjust_stmt_cost (vect_cost_for_stmt kind,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
aarch64_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@ -13758,13 +13759,14 @@ aarch64_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
aarch64_builtin_vectorization_cost (kind, vectype, misalign);
if (stmt_info && vectype && aarch64_sve_mode_p (TYPE_MODE (vectype)))
stmt_cost = aarch64_sve_adjust_stmt_cost (kind, stmt_info, vectype,
stmt_cost);
stmt_cost = aarch64_sve_adjust_stmt_cost (vinfo, kind, stmt_info,
vectype, stmt_cost);
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
if (where == vect_body && stmt_info
&& stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME */
retval = (unsigned) (count * stmt_cost);

6
gcc/config/arm/arm.c

@ -12131,7 +12131,8 @@ arm_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
arm_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
arm_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@ -12146,7 +12147,8 @@ arm_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
if (where == vect_body && stmt_info
&& stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);

6
gcc/config/i386/i386.c

@ -21878,7 +21878,8 @@ ix86_init_cost (class loop *)
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@ -22039,7 +22040,8 @@ ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
if (where == vect_body && stmt_info
&& stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);

6
gcc/config/rs6000/rs6000.c

@ -5046,7 +5046,8 @@ adjust_vectorization_cost (enum vect_cost_for_stmt kind,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
rs6000_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@ -5062,7 +5063,8 @@ rs6000_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
if (where == vect_body && stmt_info
&& stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);

2
gcc/doc/tm.texi

@ -6094,7 +6094,7 @@ all zeros. GCC can then try to branch around the instruction instead.
This hook should initialize target-specific data structures in preparation for modeling the costs of vectorizing a loop or basic block. The default allocates three unsigned integers for accumulating costs for the prologue, body, and epilogue of the loop or basic block. If @var{loop_info} is non-NULL, it identifies the loop being vectorized; otherwise a single block is being vectorized.
@end deftypefn
@deftypefn {Target Hook} unsigned TARGET_VECTORIZE_ADD_STMT_COST (void *@var{data}, int @var{count}, enum vect_cost_for_stmt @var{kind}, class _stmt_vec_info *@var{stmt_info}, int @var{misalign}, enum vect_cost_model_location @var{where})
@deftypefn {Target Hook} unsigned TARGET_VECTORIZE_ADD_STMT_COST (class vec_info *@var{}, void *@var{data}, int @var{count}, enum vect_cost_for_stmt @var{kind}, class _stmt_vec_info *@var{stmt_info}, int @var{misalign}, enum vect_cost_model_location @var{where})
This hook should update the target-specific @var{data} in response to adding @var{count} copies of the given @var{kind} of statement to a loop or basic block. The default adds the builtin vectorizer cost for the copies of the statement to the accumulator specified by @var{where}, (the prologue, body, or epilogue) and returns the amount added. The return value should be viewed as a tentative cost that may later be revised.
@end deftypefn

2
gcc/target.def

@ -2030,7 +2030,7 @@ DEFHOOK
"return value should be viewed as a tentative cost that may later be "
"revised.",
unsigned,
(void *data, int count, enum vect_cost_for_stmt kind,
(class vec_info *, void *data, int count, enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where),
default_add_stmt_cost)

2
gcc/target.h

@ -157,7 +157,7 @@ class predefined_function_abi;
/* These are defined in tree-vect-stmts.c. */
extern tree stmt_vectype (class _stmt_vec_info *);
extern bool stmt_in_inner_loop_p (class _stmt_vec_info *);
extern bool stmt_in_inner_loop_p (class vec_info *, class _stmt_vec_info *);
/* Assembler instructions for creating various kinds of integer object. */

6
gcc/targhooks.c

@ -1348,7 +1348,8 @@ default_init_cost (class loop *loop_info ATTRIBUTE_UNUSED)
it into the cost specified by WHERE, and returns the cost added. */
unsigned
default_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
default_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@ -1361,7 +1362,8 @@ default_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
if (where == vect_body && stmt_info
&& stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);

199
gcc/tree-vect-data-refs.c

@ -695,7 +695,8 @@ vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
disambiguating the loads. */
static bool
vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
vect_slp_analyze_node_dependences (vec_info *vinfo,
slp_instance instance, slp_tree node,
vec<stmt_vec_info> stores,
stmt_vec_info last_store_info)
{
@ -703,7 +704,6 @@ vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
in NODE verifying we can sink them up to the last stmt in the
group. */
stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
vec_info *vinfo = last_access_info->vinfo;
for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
{
stmt_vec_info access_info = SLP_TREE_SCALAR_STMTS (node)[k];
@ -781,7 +781,7 @@ vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
the maximum vectorization factor the data dependences allow. */
bool
vect_slp_analyze_instance_dependence (slp_instance instance)
vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
@ -794,7 +794,8 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
stmt_vec_info last_store_info = NULL;
if (store)
{
if (! vect_slp_analyze_node_dependences (instance, store, vNULL, NULL))
if (! vect_slp_analyze_node_dependences (vinfo, instance, store,
vNULL, NULL))
return false;
/* Mark stores in this instance and remember the last one. */
@ -810,7 +811,7 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
slp_tree load;
unsigned int i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, load)
if (! vect_slp_analyze_node_dependences (instance, load,
if (! vect_slp_analyze_node_dependences (vinfo, instance, load,
store
? SLP_TREE_SCALAR_STMTS (store)
: vNULL, last_store_info))
@ -831,10 +832,9 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
in STMT_INFO. */
static void
vect_record_base_alignment (stmt_vec_info stmt_info,
vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
innermost_loop_behavior *drb)
{
vec_info *vinfo = stmt_info->vinfo;
bool existed;
innermost_loop_behavior *&entry
= vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
@ -877,13 +877,13 @@ vect_record_base_alignments (vec_info *vinfo)
&& STMT_VINFO_VECTORIZABLE (stmt_info)
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
vect_record_base_alignment (stmt_info, &DR_INNERMOST (dr));
vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
/* If DR is nested in the loop that is being vectorized, we can also
record the alignment of the base wrt the outer loop. */
if (loop && nested_in_vect_loop_p (loop, stmt_info))
vect_record_base_alignment
(stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
(vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
}
}
}
@ -908,11 +908,11 @@ vect_calculate_target_alignment (dr_vec_info *dr_info)
only for trivial cases. TODO. */
static void
vect_compute_data_ref_alignment (dr_vec_info *dr_info)
vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_info = dr_info->stmt;
vec_base_alignments *base_alignments = &stmt_info->vinfo->base_alignments;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
vec_base_alignments *base_alignments = &vinfo->base_alignments;
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
tree ref = DR_REF (dr_info->dr);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
@ -930,7 +930,7 @@ vect_compute_data_ref_alignment (dr_vec_info *dr_info)
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
return;
innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
bool step_preserves_misalignment_p;
poly_uint64 vector_alignment
@ -1137,10 +1137,10 @@ vect_update_misalignment_for_peel (dr_vec_info *dr_info,
Return TRUE if DR_INFO can be handled with respect to alignment. */
static opt_result
verify_data_ref_alignment (dr_vec_info *dr_info)
verify_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
enum dr_alignment_support supportable_dr_alignment
= vect_supportable_dr_alignment (dr_info, false);
= vect_supportable_dr_alignment (vinfo, dr_info, false);
if (!supportable_dr_alignment)
return opt_result::failure_at
(dr_info->stmt->stmt,
@ -1187,7 +1187,7 @@ vect_verify_datarefs_alignment (loop_vec_info vinfo)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
opt_result res = verify_data_ref_alignment (dr_info);
opt_result res = verify_data_ref_alignment (vinfo, dr_info);
if (!res)
return res;
}
@ -1278,14 +1278,14 @@ vector_alignment_reachable_p (dr_vec_info *dr_info)
/* Calculate the cost of the memory access represented by DR_INFO. */
static void
vect_get_data_access_cost (dr_vec_info *dr_info,
vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
unsigned int *inside_cost,
unsigned int *outside_cost,
stmt_vector_for_cost *body_cost_vec,
stmt_vector_for_cost *prologue_cost_vec)
{
stmt_vec_info stmt_info = dr_info->stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
int ncopies;
if (PURE_SLP_STMT (stmt_info))
@ -1294,10 +1294,10 @@ vect_get_data_access_cost (dr_vec_info *dr_info,
ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
if (DR_IS_READ (dr_info->dr))
vect_get_load_cost (stmt_info, ncopies, true, inside_cost, outside_cost,
prologue_cost_vec, body_cost_vec, false);
vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost,
outside_cost, prologue_cost_vec, body_cost_vec, false);
else
vect_get_store_cost (stmt_info, ncopies, inside_cost, body_cost_vec);
vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@ -1315,6 +1315,7 @@ typedef struct _vect_peel_info
typedef struct _vect_peel_extended_info
{
vec_info *vinfo;
struct _vect_peel_info peel_info;
unsigned int inside_cost;
unsigned int outside_cost;
@ -1352,7 +1353,7 @@ vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
struct _vect_peel_info elem, *slot;
_vect_peel_info **new_slot;
bool supportable_dr_alignment
= vect_supportable_dr_alignment (dr_info, true);
= vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
elem.npeel = npeel;
slot = peeling_htab->find (&elem);
@ -1440,7 +1441,7 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
SET_DR_MISALIGNMENT (dr_info, 0);
else
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
vect_get_data_access_cost (dr_info, inside_cost, outside_cost,
vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost,
body_cost_vec, prologue_cost_vec);
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
}
@ -1456,8 +1457,7 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
vect_peel_info elem = *slot;
int dummy;
unsigned int inside_cost = 0, outside_cost = 0;
stmt_vec_info stmt_info = elem->dr_info->stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
stmt_vector_for_cost prologue_cost_vec, body_cost_vec,
epilogue_cost_vec;
@ -1509,6 +1509,7 @@ vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_hta
struct _vect_peel_extended_info res;
res.peel_info.dr_info = NULL;
res.vinfo = loop_vinfo;
if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
{
@ -1565,7 +1566,7 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
save_misalignment = DR_MISALIGNMENT (dr_info);
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
supportable_dr_alignment
= vect_supportable_dr_alignment (dr_info, false);
= vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
if (!supportable_dr_alignment)
@ -1753,7 +1754,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
supportable_dr_alignment = vect_supportable_dr_alignment (dr_info, true);
supportable_dr_alignment
= vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
do_peeling = vector_alignment_reachable_p (dr_info);
if (do_peeling)
{
@ -2217,7 +2219,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
}
supportable_dr_alignment
= vect_supportable_dr_alignment (dr_info, false);
= vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
if (!supportable_dr_alignment)
{
@ -2415,7 +2417,7 @@ vect_analyze_data_refs_alignment (loop_vec_info vinfo)
{
dr_vec_info *dr_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
vect_compute_data_ref_alignment (dr_info);
vect_compute_data_ref_alignment (vinfo, dr_info);
}
return opt_result::success ();
@ -2425,7 +2427,7 @@ vect_analyze_data_refs_alignment (loop_vec_info vinfo)
/* Analyze alignment of DRs of stmts in NODE. */
static bool
vect_slp_analyze_and_verify_node_alignment (slp_tree node)
vect_slp_analyze_and_verify_node_alignment (vec_info *vinfo, slp_tree node)
{
/* We vectorize from the first scalar stmt in the node unless
the node is permuted in which case we start from the first
@ -2436,12 +2438,12 @@ vect_slp_analyze_and_verify_node_alignment (slp_tree node)
first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
vect_compute_data_ref_alignment (dr_info);
vect_compute_data_ref_alignment (vinfo, dr_info);
/* For creating the data-ref pointer we need alignment of the
first element anyway. */
if (dr_info != first_dr_info)
vect_compute_data_ref_alignment (first_dr_info);
if (! verify_data_ref_alignment (dr_info))
vect_compute_data_ref_alignment (vinfo, first_dr_info);
if (! verify_data_ref_alignment (vinfo, dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@ -2459,20 +2461,21 @@ vect_slp_analyze_and_verify_node_alignment (slp_tree node)
Return FALSE if a data reference is found that cannot be vectorized. */
bool
vect_slp_analyze_and_verify_instance_alignment (slp_instance instance)
vect_slp_analyze_and_verify_instance_alignment (vec_info *vinfo,
slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_and_verify_instance_alignment");
slp_tree node;
unsigned i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
if (! vect_slp_analyze_and_verify_node_alignment (node))
if (! vect_slp_analyze_and_verify_node_alignment (vinfo, node))
return false;
node = SLP_INSTANCE_TREE (instance);
if (STMT_VINFO_DATA_REF (SLP_TREE_SCALAR_STMTS (node)[0])
&& ! vect_slp_analyze_and_verify_node_alignment
(SLP_INSTANCE_TREE (instance)))
(vinfo, SLP_INSTANCE_TREE (instance)))
return false;
return true;
@ -2486,15 +2489,15 @@ vect_slp_analyze_and_verify_instance_alignment (slp_instance instance)
Worker for vect_analyze_group_access. */
static bool
vect_analyze_group_access_1 (dr_vec_info *dr_info)
vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
{
data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
stmt_vec_info stmt_info = dr_info->stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
HOST_WIDE_INT dr_step = -1;
HOST_WIDE_INT groupsize, last_accessed_element = 1;
bool slp_impossible = false;
@ -2696,9 +2699,9 @@ vect_analyze_group_access_1 (dr_vec_info *dr_info)
Collect groups of strided stores for further use in SLP analysis. */
static bool
vect_analyze_group_access (dr_vec_info *dr_info)
vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
{
if (!vect_analyze_group_access_1 (dr_info))
if (!vect_analyze_group_access_1 (vinfo, dr_info))
{
/* Dissolve the group if present. */
stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
@ -2719,13 +2722,13 @@ vect_analyze_group_access (dr_vec_info *dr_info)
analyze groups of accesses. */
static bool
vect_analyze_data_ref_access (dr_vec_info *dr_info)
vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
{
data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
stmt_vec_info stmt_info = dr_info->stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
@ -2804,10 +2807,10 @@ vect_analyze_data_ref_access (dr_vec_info *dr_info)
if (TREE_CODE (step) != INTEGER_CST)
return (STMT_VINFO_STRIDED_P (stmt_info)
&& (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
|| vect_analyze_group_access (dr_info)));
|| vect_analyze_group_access (vinfo, dr_info)));
/* Not consecutive access - check if it's a part of interleaving group. */
return vect_analyze_group_access (dr_info);
return vect_analyze_group_access (vinfo, dr_info);
}
/* Compare two data-references DRA and DRB to group them into chunks
@ -3153,7 +3156,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo)
{
dr_vec_info *dr_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
&& !vect_analyze_data_ref_access (dr_info))
&& !vect_analyze_data_ref_access (vinfo, dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@ -3204,7 +3207,7 @@ vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
gives the worst-case number of bytes covered by the segment. */
static unsigned HOST_WIDE_INT
vect_vfa_access_size (dr_vec_info *dr_info)
vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_vinfo = dr_info->stmt;
tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
@ -3216,7 +3219,7 @@ vect_vfa_access_size (dr_vec_info *dr_info)
access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
}
if (STMT_VINFO_VEC_STMT (stmt_vinfo)
&& (vect_supportable_dr_alignment (dr_info, false)
&& (vect_supportable_dr_alignment (vinfo, dr_info, false)
== dr_explicit_realign_optimized))
{
/* We might access a full vector's worth. */
@ -3592,8 +3595,8 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
}
access_size_a = vect_vfa_access_size (dr_info_a);
access_size_b = vect_vfa_access_size (dr_info_b);
access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
align_a = vect_vfa_align (dr_info_a);
align_b = vect_vfa_align (dr_info_b);
@ -4580,7 +4583,7 @@ vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
FORNOW: We are only handling array accesses with step 1. */
tree
vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_seq *new_stmt_list,
tree offset,
tree byte_offset)
@ -4593,11 +4596,11 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
gimple_seq seq = NULL;
tree vect_ptr_type;
tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true));
tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
@ -4714,14 +4717,14 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
3. Return the pointer. */
tree
vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
class loop *at_loop, tree offset,
vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
tree aggr_type, class loop *at_loop, tree offset,
tree *initial_address, gimple_stmt_iterator *gsi,
gimple **ptr_incr, bool only_init,
tree byte_offset, tree iv_step)
{
const char *base_name;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
bool nested_in_vect_loop = false;
class loop *containing_loop = NULL;
@ -4739,7 +4742,7 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
bool insert_after;
tree indx_before_incr, indx_after_incr;
gimple *incr;
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
gcc_assert (iv_step != NULL_TREE
|| TREE_CODE (aggr_type) == ARRAY_TYPE
@ -4848,7 +4851,8 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
/* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */
new_temp = vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
new_temp = vect_create_addr_base_for_vector_ref (vinfo,
stmt_info, &new_stmt_list,
offset, byte_offset);
if (new_stmt_list)
{
@ -4875,7 +4879,7 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
{
/* Accesses to invariant addresses should be handled specially
by the caller. */
tree step = vect_dr_behavior (dr_info)->step;
tree step = vect_dr_behavior (vinfo, dr_info)->step;
gcc_assert (!integer_zerop (step));
if (iv_step == NULL_TREE)
@ -4977,7 +4981,8 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
*/
tree
bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
bump_vector_ptr (vec_info *vinfo,
tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info, tree bump)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
@ -4997,7 +5002,7 @@ bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
dataref_ptr, update);
vect_finish_stmt_generation (stmt_info, incr_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
@ -5277,7 +5282,7 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
I4: 6 14 22 30 7 15 23 31. */
void
vect_permute_store_chain (vec<tree> dr_chain,
vect_permute_store_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@ -5344,7 +5349,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_low);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect1 = data_ref;
vect2 = dr_chain[2];
@ -5355,7 +5360,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_high);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j] = data_ref;
}
}
@ -5394,7 +5399,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
vect2, perm_mask_high);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j] = high;
/* Create interleaving stmt:
@ -5404,7 +5409,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
vect2, perm_mask_low);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j+1] = low;
}
memcpy (dr_chain.address (), result_chain->address (),
@ -5465,14 +5470,14 @@ vect_permute_store_chain (vec<tree> dr_chain,
Return value - the result of the loop-header phi node. */
tree
vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree *realignment_token,
vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi, tree *realignment_token,
enum dr_alignment_support alignment_support_scheme,
tree init_addr,
class loop **at_loop)
{
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
struct data_reference *dr = dr_info->dr;
class loop *loop = NULL;
@ -5579,7 +5584,7 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (!compute_in_loop);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
ptr = vect_create_data_ref_ptr (stmt_info, vectype,
ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
loop_for_initial_load, NULL_TREE,
&init_addr, NULL, &inc, true);
if (TREE_CODE (ptr) == SSA_NAME)
@ -5626,7 +5631,8 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (!init_addr)
{
/* Generate the INIT_ADDR computation outside LOOP. */
init_addr = vect_create_addr_base_for_vector_ref (stmt_info, &stmts,
init_addr = vect_create_addr_base_for_vector_ref (vinfo,
stmt_info, &stmts,
NULL_TREE);
if (loop)
{
@ -5900,7 +5906,7 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
4th vec (E4): 3 7 11 15 19 23 27 31. */
static void
vect_permute_load_chain (vec<tree> dr_chain,
vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@ -5953,7 +5959,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_low);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
/* Create interleaving stmt (high part of):
high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
@ -5963,7 +5969,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_high);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[k] = data_ref;
}
}
@ -5998,7 +6004,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_even);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
/* data_ref = permute_odd (first_data_ref, second_data_ref); */
@ -6006,7 +6012,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_odd);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2+length/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
@ -6103,7 +6109,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
*/
static bool
vect_shift_permute_load_chain (vec<tree> dr_chain,
vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@ -6116,7 +6122,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
unsigned int i;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
unsigned HOST_WIDE_INT nelt, vf;
if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
@ -6205,26 +6211,26 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, first_vect,
perm2_mask1);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[0] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
second_vect, second_vect,
perm2_mask2);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[1] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], shift1_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2 + length/2] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], select_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
@ -6321,7 +6327,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
dr_chain[k], dr_chain[k],
perm3_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
@ -6331,7 +6337,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[k % 3], vect[(k + 1) % 3],
shift1_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect_shift[k] = data_ref;
}
@ -6342,7 +6348,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
vect_shift[(4 - k) % 3],
vect_shift[(3 - k) % 3],
shift2_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
@ -6351,13 +6357,13 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
vect[0], shift3_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[nelt % 3] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
vect[1], shift4_mask);
vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[0] = data_ref;
return true;
}
@ -6372,7 +6378,8 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
*/
void
vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
vec<tree> dr_chain,
int size, gimple_stmt_iterator *gsi)
{
machine_mode mode;
@ -6389,10 +6396,11 @@ vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
|| pow2p_hwi (size)
|| !vect_shift_permute_load_chain (dr_chain, size, stmt_info,
|| !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
gsi, &result_chain))
vect_permute_load_chain (dr_chain, size, stmt_info, gsi, &result_chain);
vect_record_grouped_load_vectors (stmt_info, result_chain);
vect_permute_load_chain (vinfo, dr_chain,
size, stmt_info, gsi, &result_chain);
vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
result_chain.release ();
}
@ -6401,10 +6409,9 @@ vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
for each vector to the associated scalar statement. */
void
vect_record_grouped_load_vectors (stmt_vec_info stmt_info,
vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
vec<tree> result_chain)
{
vec_info *vinfo = stmt_info->vinfo;
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
unsigned int i, gap_count;
tree tmp_data_ref;
@ -6493,14 +6500,14 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
alignment. */
enum dr_alignment_support
vect_supportable_dr_alignment (dr_vec_info *dr_info,
vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
bool check_aligned_accesses)
{
data_reference *dr = dr_info->dr;
stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
machine_mode mode = TYPE_MODE (vectype);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *vect_loop = NULL;
bool nested_in_vect_loop = false;

6
gcc/tree-vect-loop-manip.c

@ -1568,7 +1568,8 @@ get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
tree offset = (negative
? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1)
: size_zero_node);
tree start_addr = vect_create_addr_base_for_vector_ref (stmt_info, seq,
tree start_addr = vect_create_addr_base_for_vector_ref (loop_vinfo,
stmt_info, seq,
offset);
tree type = unsigned_type_for (TREE_TYPE (start_addr));
if (target_align.is_constant (&target_align_c))
@ -3057,7 +3058,8 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo,
/* create: addr_tmp = (int)(address_of_first_vector) */
addr_base =
vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
vect_create_addr_base_for_vector_ref (loop_vinfo,
stmt_info, &new_stmt_list,
offset);
if (new_stmt_list != NULL)
gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list);

221
gcc/tree-vect-loop.c

@ -161,7 +161,7 @@ static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
may already be set for general statements (not just data refs). */
static opt_result
vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
bool vectype_maybe_set_p,
poly_uint64 *vf)
{
@ -177,7 +177,8 @@ vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
}
tree stmt_vectype, nunits_vectype;
opt_result res = vect_get_vector_types_for_stmt (stmt_info, &stmt_vectype,
opt_result res = vect_get_vector_types_for_stmt (vinfo, stmt_info,
&stmt_vectype,
&nunits_vectype);
if (!res)
return res;
@ -207,13 +208,13 @@ vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
or false if something prevented vectorization. */
static opt_result
vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
vect_determine_vf_for_stmt (vec_info *vinfo,
stmt_vec_info stmt_info, poly_uint64 *vf)
{
vec_info *vinfo = stmt_info->vinfo;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
stmt_info->stmt);
opt_result res = vect_determine_vf_for_stmt_1 (stmt_info, false, vf);
opt_result res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, false, vf);
if (!res)
return res;
@ -232,7 +233,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern def stmt: %G",
def_stmt_info->stmt);
res = vect_determine_vf_for_stmt_1 (def_stmt_info, true, vf);
res = vect_determine_vf_for_stmt_1 (vinfo, def_stmt_info, true, vf);
if (!res)
return res;
}
@ -241,7 +242,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern statement: %G",
stmt_info->stmt);
res = vect_determine_vf_for_stmt_1 (stmt_info, true, vf);
res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, true, vf);
if (!res)
return res;
}
@ -343,7 +344,8 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
{
stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
opt_result res
= vect_determine_vf_for_stmt (stmt_info, &vectorization_factor);
= vect_determine_vf_for_stmt (loop_vinfo,
stmt_info, &vectorization_factor);
if (!res)
return res;
}
@ -440,9 +442,8 @@ vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
this function would then return true for x_2. */
static bool
vect_inner_phi_in_double_reduction_p (stmt_vec_info stmt_info, gphi *phi)
vect_inner_phi_in_double_reduction_p (loop_vec_info loop_vinfo, gphi *phi)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
use_operand_p use_p;
ssa_op_iter op_iter;
FOR_EACH_PHI_ARG (use_p, phi, op_iter, SSA_OP_USE)
@ -505,7 +506,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
}
if (!access_fn
|| vect_inner_phi_in_double_reduction_p (stmt_vinfo, phi)
|| vect_inner_phi_in_double_reduction_p (loop_vinfo, phi)
|| !vect_is_simple_iv_evolution (loop->num, access_fn, &init, &step)
|| (LOOP_VINFO_LOOP (loop_vinfo) != loop
&& TREE_CODE (step) != INTEGER_CST))
@ -1122,7 +1123,7 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (target_cost_data, si->count,
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
si->kind, si->stmt_info, si->misalign,
vect_body);
unsigned dummy, body_cost = 0;
@ -1529,7 +1530,8 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
|| (STMT_VINFO_DEF_TYPE (stmt_info)
== vect_double_reduction_def))
&& !vectorizable_lc_phi (stmt_info, NULL, NULL))
&& !vectorizable_lc_phi (loop_vinfo,
stmt_info, NULL, NULL))
return opt_result::failure_at (phi, "unsupported phi\n");
}
@ -1551,21 +1553,24 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
need_to_vectorize = true;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
&& ! PURE_SLP_STMT (stmt_info))
ok = vectorizable_induction (stmt_info, NULL, NULL, NULL,
ok = vectorizable_induction (loop_vinfo,
stmt_info, NULL, NULL, NULL,
&cost_vec);
else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
|| (STMT_VINFO_DEF_TYPE (stmt_info)
== vect_double_reduction_def)
|| STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
&& ! PURE_SLP_STMT (stmt_info))
ok = vectorizable_reduction (stmt_info, NULL, NULL, &cost_vec);
ok = vectorizable_reduction (loop_vinfo,
stmt_info, NULL, NULL, &cost_vec);
}
/* SLP PHIs are tested by vect_slp_analyze_node_operations. */
if (ok
&& STMT_VINFO_LIVE_P (stmt_info)
&& !PURE_SLP_STMT (stmt_info))
ok = vectorizable_live_operation (stmt_info, NULL, NULL, NULL,
ok = vectorizable_live_operation (loop_vinfo,
stmt_info, NULL, NULL, NULL,
-1, false, &cost_vec);
if (!ok)
@ -1582,7 +1587,8 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
if (!gimple_clobber_p (stmt))
{
opt_result res
= vect_analyze_stmt (loop_vinfo->lookup_stmt (stmt),
= vect_analyze_stmt (loop_vinfo,
loop_vinfo->lookup_stmt (stmt),
&need_to_vectorize,
NULL, NULL, &cost_vec);
if (!res)
@ -1591,7 +1597,7 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
}
} /* bbs */
add_stmt_costs (loop_vinfo->target_cost_data, &cost_vec);
add_stmt_costs (loop_vinfo, loop_vinfo->target_cost_data, &cost_vec);
/* All operations in the loop are either irrelevant (deal with loop
control, or dead), or only used outside the loop and can be moved
@ -3397,8 +3403,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
{
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
(void) add_stmt_cost (target_cost_data, len, vector_stmt, NULL, 0,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
NULL, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@ -3410,13 +3416,13 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
{
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
(void) add_stmt_cost (target_cost_data, len, vector_stmt, NULL, 0,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
NULL, 0, vect_prologue);
len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
if (len)
/* Count LEN - 1 ANDs and LEN comparisons. */
(void) add_stmt_cost (target_cost_data, len * 2 - 1, scalar_stmt,
NULL, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, len * 2 - 1,
scalar_stmt, NULL, 0, vect_prologue);
len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
if (len)
{
@ -3426,8 +3432,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
for (unsigned int i = 0; i < len; ++i)
if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
nstmts += 1;
(void) add_stmt_cost (target_cost_data, nstmts, scalar_stmt,
NULL, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, nstmts,
scalar_stmt, NULL, 0, vect_prologue);
}
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
@ -3439,8 +3445,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
{
/* FIXME: Make cost depend on complexity of individual check. */
(void) add_stmt_cost (target_cost_data, 1, vector_stmt, NULL, 0,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, vector_stmt,
NULL, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@ -3448,8 +3454,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
}
if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
(void) add_stmt_cost (target_cost_data, 1, cond_branch_taken, NULL, 0,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
NULL, 0, vect_prologue);
/* Count statements in scalar loop. Using this as scalar cost for a single
iteration for now.
@ -3484,7 +3490,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (target_cost_data, si->count,
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
si->kind, si->stmt_info, si->misalign,
vect_epilogue);
}
@ -3510,9 +3516,11 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
simpler and safer to use the worst-case cost; if this ends up
being the tie-breaker between vectorizing or not, then it's
probably better not to vectorize. */
(void) add_stmt_cost (target_cost_data, num_masks, vector_stmt,
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks, vector_stmt,
NULL, 0, vect_prologue);
(void) add_stmt_cost (target_cost_data, num_masks - 1, vector_stmt,
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks - 1, vector_stmt,
NULL, 0, vect_body);
}
else if (npeel < 0)
@ -3534,23 +3542,25 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
branch per peeled loop. Even if scalar loop iterations are known,
vector iterations are not known since peeled prologue iterations are
not known. Hence guards remain the same. */
(void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
NULL, 0, vect_prologue);
(void) add_stmt_cost (target_cost_data, 1, cond_branch_not_taken,
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
NULL, 0, vect_prologue);
(void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
NULL, 0, vect_epilogue);
(void) add_stmt_cost (target_cost_data, 1, cond_branch_not_taken,
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
NULL, 0, vect_epilogue);
stmt_info_for_cost *si;
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
{
(void) add_stmt_cost (target_cost_data,
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_prologue,
si->kind, si->stmt_info, si->misalign,
vect_prologue);
(void) add_stmt_cost (target_cost_data,
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_epilogue,
si->kind, si->stmt_info, si->misalign,
vect_epilogue);
@ -3575,11 +3585,13 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
&epilogue_cost_vec);
FOR_EACH_VEC_ELT (prologue_cost_vec, j, si)
(void) add_stmt_cost (data, si->count, si->kind, si->stmt_info,
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
si->misalign, vect_prologue);
FOR_EACH_VEC_ELT (epilogue_cost_vec, j, si)
(void) add_stmt_cost (data, si->count, si->kind, si->stmt_info,
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
si->misalign, vect_epilogue);
prologue_cost_vec.release ();
@ -3910,7 +3922,8 @@ have_whole_vector_shift (machine_mode mode)
the loop, and the epilogue code that must be generated. */
static void
vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
vect_model_reduction_cost (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, internal_fn reduc_fn,
vect_reduction_type reduction_type,
int ncopies, stmt_vector_for_cost *cost_vec)
{
@ -3919,7 +3932,6 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
optab optab;
tree vectype;
machine_mode mode;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = NULL;
if (loop_vinfo)
@ -4148,11 +4160,11 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
A cost model should help decide between these two schemes. */
static tree
get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
get_initial_def_for_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_vinfo,
enum tree_code code, tree init_val,
tree *adjustment_def)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree scalar_type = TREE_TYPE (init_val);
tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
@ -4252,14 +4264,14 @@ get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
value will not change the result. */
static void
get_initial_defs_for_reduction (slp_tree slp_node,
get_initial_defs_for_reduction (vec_info *vinfo,
slp_tree slp_node,
vec<tree> *vec_oprnds,
unsigned int number_of_vectors,
bool reduc_chain, tree neutral_op)
{
vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
stmt_vec_info stmt_vinfo = stmts[0];
vec_info *vinfo = stmt_vinfo->vinfo;
unsigned HOST_WIDE_INT nunits;
unsigned j, number_of_places_left_in_vector;
tree vector_type;
@ -4372,7 +4384,7 @@ get_initial_defs_for_reduction (slp_tree slp_node,
the stmt_vec_info the meta information is stored on. */
stmt_vec_info
info_for_reduction (stmt_vec_info stmt_info)
info_for_reduction (vec_info *vinfo, stmt_vec_info stmt_info)
{
stmt_info = vect_orig_stmt (stmt_info);
gcc_assert (STMT_VINFO_REDUC_DEF (stmt_info));
@ -4388,7 +4400,7 @@ info_for_reduction (stmt_vec_info stmt_info)
{
edge pe = loop_preheader_edge (gimple_bb (phi)->loop_father);
stmt_vec_info info
= stmt_info->vinfo->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi, pe));
= vinfo->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi, pe));
if (info && STMT_VINFO_DEF_TYPE (info) == vect_double_reduction_def)
stmt_info = info;
}
@ -4443,13 +4455,13 @@ info_for_reduction (stmt_vec_info stmt_info)
*/
static void
vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
slp_tree slp_node,
slp_instance slp_node_instance)
{
stmt_vec_info reduc_info = info_for_reduction (stmt_info);
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
/* For double reductions we need to get at the inner loop reduction
stmt which has the meta info attached. Our stmt_info is that of the
loop-closed PHI of the inner loop which we remember as
@ -5659,7 +5671,8 @@ get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
that should be used to control the operation in a fully-masked loop. */
static bool
vectorize_fold_left_reduction (stmt_vec_info stmt_info,
vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
gimple *reduc_def_stmt,
@ -5667,7 +5680,6 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
tree ops[3], tree vectype_in,
int reduc_index, vec_loop_masks *masks)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
stmt_vec_info new_stmt_info = NULL;
@ -5695,7 +5707,7 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
if (slp_node)
{
auto_vec<vec<tree> > vec_defs (2);
vect_get_slp_defs (slp_node, &vec_defs);
vect_get_slp_defs (loop_vinfo, slp_node, &vec_defs);
vec_oprnds0.safe_splice (vec_defs[1 - reduc_index]);
vec_defs[0].release ();
vec_defs[1].release ();
@ -5704,7 +5716,8 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
}
else
{
tree loop_vec_def0 = vect_get_vec_def_for_operand (op0, stmt_info);
tree loop_vec_def0 = vect_get_vec_def_for_operand (loop_vinfo,
op0, stmt_info);
vec_oprnds0.create (1);
vec_oprnds0.quick_push (loop_vec_def0);
scalar_dest_def_info = stmt_info;
@ -5782,11 +5795,13 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
if (i == vec_num - 1)
{
gimple_set_lhs (new_stmt, scalar_dest);
new_stmt_info = vect_finish_replace_stmt (scalar_dest_def_info,
new_stmt_info = vect_finish_replace_stmt (loop_vinfo,
scalar_dest_def_info,
new_stmt);
}
else
new_stmt_info = vect_finish_stmt_generation (scalar_dest_def_info,
new_stmt_info = vect_finish_stmt_generation (loop_vinfo,
scalar_dest_def_info,
new_stmt, gsi);
if (slp_node)
@ -5953,13 +5968,13 @@ build_vect_cond_expr (enum tree_code code, tree vop[3], tree mask,
does *NOT* necessarily hold for reduction patterns. */
bool
vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
vectorizable_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, slp_tree slp_node,
slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec)
{
tree scalar_dest;
tree vectype_in = NULL_TREE;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
stmt_vec_info cond_stmt_vinfo = NULL;
@ -5981,7 +5996,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
return false;
/* The stmt we store reduction analysis meta on. */
stmt_vec_info reduc_info = info_for_reduction (stmt_info);
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
reduc_info->is_reduc_info = true;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
@ -6714,8 +6729,8 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
else
vec_num = 1;
vect_model_reduction_cost (stmt_info, reduc_fn, reduction_type, ncopies,
cost_vec);
vect_model_reduction_cost (loop_vinfo, stmt_info, reduc_fn,
reduction_type, ncopies, cost_vec);
if (dump_enabled_p ()
&& reduction_type == FOLD_LEFT_REDUCTION)
dump_printf_loc (MSG_NOTE, vect_location,
@ -6779,18 +6794,18 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
value. */
bool
vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vect_transform_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
int j;
int vec_num;
stmt_vec_info reduc_info = info_for_reduction (stmt_info);
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (nested_in_vect_loop_p (loop, stmt_info))
@ -6865,7 +6880,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
return vectorize_fold_left_reduction
(stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
(loop_vinfo, stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
reduc_fn, ops, vectype_in, reduc_index, masks);
}
@ -6898,7 +6913,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Get vec defs for all the operands except the reduction index,
ensuring the ordering of the ops in the vector is kept. */
auto_vec<vec<tree>, 3> vec_defs;
vect_get_slp_defs (slp_node, &vec_defs);
vect_get_slp_defs (loop_vinfo, slp_node, &vec_defs);
vec_oprnds0.safe_splice (vec_defs[0]);
vec_defs[0].release ();
vec_oprnds1.safe_splice (vec_defs[1]);
@ -6912,12 +6927,12 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else
{
vec_oprnds0.quick_push
(vect_get_vec_def_for_operand (ops[0], stmt_info));
(vect_get_vec_def_for_operand (loop_vinfo, ops[0], stmt_info));
vec_oprnds1.quick_push
(vect_get_vec_def_for_operand (ops[1], stmt_info));
(vect_get_vec_def_for_operand (loop_vinfo, ops[1], stmt_info));
if (op_type == ternary_op)
vec_oprnds2.quick_push
(vect_get_vec_def_for_operand (ops[2], stmt_info));
(vect_get_vec_def_for_operand (loop_vinfo, ops[2], stmt_info));
}
}
else
@ -6970,7 +6985,8 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
= vect_finish_stmt_generation (stmt_info, call, gsi);
= vect_finish_stmt_generation (loop_vinfo,
stmt_info, call, gsi);
}
else
{
@ -6990,7 +7006,8 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
= vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
= vect_finish_stmt_generation (loop_vinfo,
stmt_info, new_stmt, gsi);
}
if (slp_node)
@ -7017,11 +7034,11 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Transform phase of a cycle PHI. */
bool
vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
vect_transform_cycle_phi (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
slp_tree slp_node, slp_instance slp_node_instance)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
@ -7038,7 +7055,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info);
stmt_vec_info reduc_info = info_for_reduction (stmt_info);
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
@ -7088,7 +7105,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
= neutral_op_for_slp_reduction (slp_node, vectype_out,
STMT_VINFO_REDUC_CODE (reduc_info),
first != NULL);
get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
get_initial_defs_for_reduction (loop_vinfo, slp_node_instance->reduc_phis,
&vec_initial_defs, vec_num,
first != NULL, neutral_op);
}
@ -7122,7 +7139,8 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
{
/* Do not use an adjustment def as that case is not supported
correctly if ncopies is not one. */
vec_initial_def = vect_get_vec_def_for_operand (initial_def,
vec_initial_def = vect_get_vec_def_for_operand (loop_vinfo,
initial_def,
reduc_stmt_info);
}
else
@ -7133,7 +7151,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
adjustment_defp = NULL;
vec_initial_def
= get_initial_def_for_reduction (reduc_stmt_info, code,
= get_initial_def_for_reduction (loop_vinfo, reduc_stmt_info, code,
initial_def, adjustment_defp);
STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info) = adjustment_def;
}
@ -7181,10 +7199,10 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
/* Vectorizes LC PHIs. */
bool
vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
vectorizable_lc_phi (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
slp_tree slp_node)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
if (!loop_vinfo
|| !is_a <gphi *> (stmt_info->stmt)
|| gimple_phi_num_args (stmt_info->stmt) != 1)
@ -7206,7 +7224,8 @@ vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
edge e = single_pred_edge (bb);
tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
vec<tree> vec_oprnds = vNULL;
vect_get_vec_defs (gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE,
vect_get_vec_defs (loop_vinfo,
gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE,
stmt_info, &vec_oprnds, NULL, slp_node);
if (slp_node)
{
@ -7294,12 +7313,12 @@ vect_worthwhile_without_simd_p (vec_info *vinfo, tree_code code)
Return true if STMT_INFO is vectorizable in this way. */
bool
vectorizable_induction (stmt_vec_info stmt_info,
vectorizable_induction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
unsigned ncopies;
bool nested_in_vect_loop = false;
@ -7494,10 +7513,11 @@ vectorizable_induction (stmt_vec_info stmt_info,
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
expr, step_expr);
if (! CONSTANT_CLASS_P (new_name))
new_name = vect_init_vector (stmt_info, new_name,
new_name = vect_init_vector (loop_vinfo, stmt_info, new_name,
TREE_TYPE (step_expr), NULL);
new_vec = build_vector_from_val (step_vectype, new_name);
vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
vec_step = vect_init_vector (loop_vinfo, stmt_info,
new_vec, step_vectype, NULL);
/* Now generate the IVs. */
unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
@ -7568,10 +7588,11 @@ vectorizable_induction (stmt_vec_info stmt_info,
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
expr, step_expr);
if (! CONSTANT_CLASS_P (new_name))
new_name = vect_init_vector (stmt_info, new_name,
new_name = vect_init_vector (loop_vinfo, stmt_info, new_name,
TREE_TYPE (step_expr), NULL);
new_vec = build_vector_from_val (step_vectype, new_name);
vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
vec_step = vect_init_vector (loop_vinfo, stmt_info, new_vec,
step_vectype, NULL);
for (; ivn < nvects; ++ivn)
{
gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt;
@ -7606,7 +7627,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
/* iv_loop is nested in the loop to be vectorized. init_expr had already
been created during vectorization of previous stmts. We obtain it
from the STMT_VINFO_VEC_STMT of the defining stmt. */
vec_init = vect_get_vec_def_for_operand (init_expr, stmt_info);
vec_init = vect_get_vec_def_for_operand (loop_vinfo,
init_expr, stmt_info);
/* If the initial value is not of proper type, convert it. */
if (!useless_type_conversion_p (vectype, TREE_TYPE (vec_init)))
{
@ -7709,7 +7731,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
gcc_assert (CONSTANT_CLASS_P (new_name)
|| TREE_CODE (new_name) == SSA_NAME);
new_vec = build_vector_from_val (step_vectype, t);
vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
vec_step = vect_init_vector (loop_vinfo, stmt_info,
new_vec, step_vectype, NULL);
/* Create the following def-use cycle:
@ -7778,7 +7801,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
gcc_assert (CONSTANT_CLASS_P (new_name)
|| TREE_CODE (new_name) == SSA_NAME);
new_vec = build_vector_from_val (step_vectype, t);
vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
vec_step = vect_init_vector (loop_vinfo, stmt_info,
new_vec, step_vectype, NULL);
vec_def = induc_def;
prev_stmt_vinfo = induction_phi_info;
@ -7847,13 +7871,13 @@ vectorizable_induction (stmt_vec_info stmt_info,
it can be supported. */
bool
vectorizable_live_operation (stmt_vec_info stmt_info,
vectorizable_live_operation (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
slp_tree slp_node, slp_instance slp_node_instance,
int slp_index, bool vec_stmt_p,
stmt_vector_for_cost *)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
imm_use_iterator imm_iter;
tree lhs, lhs_type, bitsize, vec_bitsize;
@ -7885,12 +7909,12 @@ vectorizable_live_operation (stmt_vec_info stmt_info,
else if (slp_index != 0)
return true;
}
stmt_vec_info reduc_info = info_for_reduction (stmt_info);
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (STMT_VINFO_REDUC_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
|| STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
return true;
vect_create_epilog_for_reduction (stmt_info, slp_node,
vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
slp_node_instance);
return true;
}
@ -8371,7 +8395,7 @@ vect_transform_loop_stmt (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
if (vect_transform_stmt (stmt_info, gsi, NULL, NULL))
if (vect_transform_stmt (loop_vinfo, stmt_info, gsi, NULL, NULL))
*seen_store = stmt_info;
}
@ -8730,7 +8754,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform phi.\n");
vect_transform_stmt (stmt_info, NULL, NULL, NULL);
vect_transform_stmt (loop_vinfo, stmt_info, NULL, NULL, NULL);
}
}
@ -8781,7 +8805,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
/* Interleaving. If IS_STORE is TRUE, the
vectorization of the interleaving chain was
completed - free all the stores in the chain. */
vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store));
vect_remove_stores (loop_vinfo,
DR_GROUP_FIRST_ELEMENT (seen_store));
else
/* Free the attached stmt_vec_info and remove the stmt. */
loop_vinfo->remove_stmt (stmt_info);

474
gcc/tree-vect-patterns.c

File diff suppressed because it is too large Load Diff

132
gcc/tree-vect-slp.c

@ -627,10 +627,10 @@ vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype)
Used only for BB vectorization. */
static bool
vect_update_all_shared_vectypes (vec<stmt_vec_info> stmts)
vect_update_all_shared_vectypes (vec_info *vinfo, vec<stmt_vec_info> stmts)
{
tree vectype, nunits_vectype;
if (!vect_get_vector_types_for_stmt (stmts[0], &vectype,
if (!vect_get_vector_types_for_stmt (vinfo, stmts[0], &vectype,
&nunits_vectype, stmts.length ()))
return false;
@ -686,7 +686,8 @@ compatible_calls_p (gcall *call1, gcall *call2)
vect_build_slp_tree. */
static bool
vect_record_max_nunits (stmt_vec_info stmt_info, unsigned int group_size,
vect_record_max_nunits (vec_info *vinfo, stmt_vec_info stmt_info,
unsigned int group_size,
tree vectype, poly_uint64 *max_nunits)
{
if (!vectype)
@ -703,7 +704,7 @@ vect_record_max_nunits (stmt_vec_info stmt_info, unsigned int group_size,
before adjusting *max_nunits for basic-block vectorization. */
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
unsigned HOST_WIDE_INT const_nunits;
if (STMT_VINFO_BB_VINFO (stmt_info)
if (is_a <bb_vec_info> (vinfo)
&& (!nunits.is_constant (&const_nunits)
|| const_nunits > group_size))
{
@ -764,7 +765,7 @@ vect_two_operations_perm_ok_p (vec<stmt_vec_info> stmts,
to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
static bool
vect_build_slp_tree_1 (unsigned char *swap,
vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
vec<stmt_vec_info> stmts, unsigned int group_size,
poly_uint64 *max_nunits, bool *matches,
bool *two_operators)
@ -789,7 +790,6 @@ vect_build_slp_tree_1 (unsigned char *swap,
stmt_vec_info stmt_info;
FOR_EACH_VEC_ELT (stmts, i, stmt_info)
{
vec_info *vinfo = stmt_info->vinfo;
gimple *stmt = stmt_info->stmt;
swap[i] = 0;
matches[i] = false;
@ -822,10 +822,10 @@ vect_build_slp_tree_1 (unsigned char *swap,
}
tree nunits_vectype;
if (!vect_get_vector_types_for_stmt (stmt_info, &vectype,
if (!vect_get_vector_types_for_stmt (vinfo, stmt_info, &vectype,
&nunits_vectype, group_size)
|| (nunits_vectype
&& !vect_record_max_nunits (stmt_info, group_size,
&& !vect_record_max_nunits (vinfo, stmt_info, group_size,
nunits_vectype, max_nunits)))
{
/* Fatal mismatch. */
@ -1256,7 +1256,8 @@ vect_build_slp_tree_2 (vec_info *vinfo,
{
tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
if (!vect_record_max_nunits (stmt_info, group_size, vectype, max_nunits))
if (!vect_record_max_nunits (vinfo, stmt_info, group_size, vectype,
max_nunits))
return NULL;
vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
@ -1288,7 +1289,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
bool two_operators = false;
unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
if (!vect_build_slp_tree_1 (swap, stmts, group_size,
if (!vect_build_slp_tree_1 (vinfo, swap, stmts, group_size,
&this_max_nunits, matches, &two_operators))
return NULL;
@ -1398,7 +1399,8 @@ vect_build_slp_tree_2 (vec_info *vinfo,
if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def)
break;
if (!grandchild
&& vect_update_all_shared_vectypes (oprnd_info->def_stmts))
&& vect_update_all_shared_vectypes (vinfo,
oprnd_info->def_stmts))
{
/* Roll back. */
this_tree_size = old_tree_size;
@ -1440,7 +1442,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
scalar version. */
&& !is_pattern_stmt_p (stmt_info)
&& !oprnd_info->any_pattern
&& vect_update_all_shared_vectypes (oprnd_info->def_stmts))
&& vect_update_all_shared_vectypes (vinfo, oprnd_info->def_stmts))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@ -1540,7 +1542,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
break;
if (!grandchild
&& (vect_update_all_shared_vectypes
(oprnd_info->def_stmts)))
(vinfo, oprnd_info->def_stmts)))
{
/* Roll back. */
this_tree_size = old_tree_size;
@ -1922,7 +1924,7 @@ vect_gather_slp_loads (slp_instance inst, slp_tree node)
SLP_INSTN are supported. */
static bool
vect_supported_load_permutation_p (slp_instance slp_instn)
vect_supported_load_permutation_p (vec_info *vinfo, slp_instance slp_instn)
{
unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_instn);
unsigned int i, j, k, next;
@ -1966,7 +1968,7 @@ vect_supported_load_permutation_p (slp_instance slp_instn)
/* In basic block vectorization we allow any subchain of an interleaving
chain.
FORNOW: not supported in loop SLP because of realignment compications. */
if (STMT_VINFO_BB_VINFO (stmt_info))
if (is_a <bb_vec_info> (vinfo))
{
/* Check whether the loads in an instance form a subchain and thus
no permutation is necessary. */
@ -2015,7 +2017,7 @@ vect_supported_load_permutation_p (slp_instance slp_instn)
/* Verify the permutation can be generated. */
vec<tree> tem;
unsigned n_perms;
if (!vect_transform_slp_perm_load (node, tem, NULL,
if (!vect_transform_slp_perm_load (vinfo, node, tem, NULL,
1, slp_instn, true, &n_perms))
{
if (dump_enabled_p ())
@ -2038,10 +2040,10 @@ vect_supported_load_permutation_p (slp_instance slp_instn)
poly_uint64 test_vf
= force_common_multiple (SLP_INSTANCE_UNROLLING_FACTOR (slp_instn),
LOOP_VINFO_VECT_FACTOR
(STMT_VINFO_LOOP_VINFO (stmt_info)));
(as_a <loop_vec_info> (vinfo)));
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node)
if (node->load_permutation.exists ()
&& !vect_transform_slp_perm_load (node, vNULL, NULL, test_vf,
&& !vect_transform_slp_perm_load (vinfo, node, vNULL, NULL, test_vf,
slp_instn, true, &n_perms))
return false;
@ -2321,7 +2323,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
if (loads_permuted)
{
if (!vect_supported_load_permutation_p (new_instance))
if (!vect_supported_load_permutation_p (vinfo, new_instance))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@ -2569,7 +2571,8 @@ vect_make_slp_decision (loop_vec_info loop_vinfo)
can't be SLPed) in the tree rooted at NODE. Mark such stmts as HYBRID. */
static void
vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype,
vect_detect_hybrid_slp_stmts (loop_vec_info loop_vinfo, slp_tree node,
unsigned i, slp_vect_type stype,
hash_map<slp_tree, unsigned> &visited)
{
stmt_vec_info stmt_vinfo = SLP_TREE_SCALAR_STMTS (node)[i];
@ -2577,7 +2580,6 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype,
gimple *use_stmt;
stmt_vec_info use_vinfo;
slp_tree child;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
int j;
/* We need to union stype over the incoming graph edges but we still
@ -2637,7 +2639,7 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype,
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
if (SLP_TREE_DEF_TYPE (child) != vect_external_def
&& SLP_TREE_DEF_TYPE (child) != vect_constant_def)
vect_detect_hybrid_slp_stmts (child, i, stype, visited);
vect_detect_hybrid_slp_stmts (loop_vinfo, child, i, stype, visited);
}
/* Helpers for vect_detect_hybrid_slp walking pattern stmt uses. */
@ -2730,7 +2732,8 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
if (j < SLP_INSTANCE_GROUP_SIZE (instance))
{
any = true;
vect_detect_hybrid_slp_stmts (SLP_INSTANCE_TREE (instance),
vect_detect_hybrid_slp_stmts (loop_vinfo,
SLP_INSTANCE_TREE (instance),
j, pure_slp, visited);
}
if (!any)
@ -2820,7 +2823,8 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
}
bool dummy;
return vect_analyze_stmt (stmt_info, &dummy, node, node_instance, cost_vec);
return vect_analyze_stmt (vinfo, stmt_info, &dummy,
node, node_instance, cost_vec);
}
/* Try to build NODE from scalars, returning true on success.
@ -2987,7 +2991,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
visited.add (*x);
i++;
add_stmt_costs (vinfo->target_cost_data, &cost_vec);
add_stmt_costs (vinfo, vinfo->target_cost_data, &cost_vec);
cost_vec.release ();
}
}
@ -3001,7 +3005,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
update LIFE according to uses of NODE. */
static void
vect_bb_slp_scalar_cost (basic_block bb,
vect_bb_slp_scalar_cost (vec_info *vinfo, basic_block bb,
slp_tree node, vec<bool, va_heap> *life,
stmt_vector_for_cost *cost_vec,
hash_set<slp_tree> &visited)
@ -3016,7 +3020,6 @@ vect_bb_slp_scalar_cost (basic_block bb,
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
{
gimple *stmt = stmt_info->stmt;
vec_info *vinfo = stmt_info->vinfo;
ssa_op_iter op_iter;
def_operand_p def_p;
@ -3074,7 +3077,7 @@ vect_bb_slp_scalar_cost (basic_block bb,
/* Do not directly pass LIFE to the recursive call, copy it to
confine changes in the callee to the current child/subtree. */
subtree_life.safe_splice (*life);
vect_bb_slp_scalar_cost (bb, child, &subtree_life, cost_vec,
vect_bb_slp_scalar_cost (vinfo, bb, child, &subtree_life, cost_vec,
visited);
subtree_life.truncate (0);
}
@ -3100,12 +3103,12 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
{
auto_vec<bool, 20> life;
life.safe_grow_cleared (SLP_INSTANCE_GROUP_SIZE (instance));
vect_bb_slp_scalar_cost (BB_VINFO_BB (bb_vinfo),
vect_bb_slp_scalar_cost (bb_vinfo, BB_VINFO_BB (bb_vinfo),
SLP_INSTANCE_TREE (instance),
&life, &scalar_costs, visited);
}
void *target_cost_data = init_cost (NULL);
add_stmt_costs (target_cost_data, &scalar_costs);
add_stmt_costs (bb_vinfo, target_cost_data, &scalar_costs);
scalar_costs.release ();
unsigned dummy;
finish_cost (target_cost_data, &dummy, &scalar_cost, &dummy);
@ -3258,8 +3261,8 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal)
dependence in the SLP instances. */
for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
{
if (! vect_slp_analyze_and_verify_instance_alignment (instance)
|| ! vect_slp_analyze_instance_dependence (instance))
if (! vect_slp_analyze_and_verify_instance_alignment (bb_vinfo, instance)
|| ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
{
slp_tree node = SLP_INSTANCE_TREE (instance);
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
@ -3497,7 +3500,8 @@ vect_slp_bb (basic_block bb)
/* Return 1 if vector type STMT_VINFO is a boolean vector. */
static bool
vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
vect_mask_constant_operand_p (vec_info *vinfo,
stmt_vec_info stmt_vinfo, unsigned op_num)
{
enum tree_code code = gimple_expr_code (stmt_vinfo->stmt);
tree op, vectype;
@ -3510,7 +3514,7 @@ vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
gassign *stmt = as_a <gassign *> (stmt_vinfo->stmt);
op = gimple_assign_rhs1 (stmt);
if (!vect_is_simple_use (op, stmt_vinfo->vinfo, &dt, &vectype))
if (!vect_is_simple_use (op, vinfo, &dt, &vectype))
gcc_unreachable ();
return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype);
@ -3534,7 +3538,7 @@ vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
op = TREE_OPERAND (cond, 0);
}
if (!vect_is_simple_use (op, stmt_vinfo->vinfo, &dt, &vectype))
if (!vect_is_simple_use (op, vinfo, &dt, &vectype))
gcc_unreachable ();
return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype);
@ -3663,12 +3667,12 @@ duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
operands. */
static void
vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
vect_get_constant_vectors (vec_info *vinfo,
slp_tree slp_node, unsigned op_num,
vec<tree> *vec_oprnds)
{
slp_tree op_node = SLP_TREE_CHILDREN (slp_node)[op_num];
stmt_vec_info stmt_vinfo = SLP_TREE_SCALAR_STMTS (slp_node)[0];
vec_info *vinfo = stmt_vinfo->vinfo;
unsigned HOST_WIDE_INT nunits;
tree vec_cst;
unsigned j, number_of_places_left_in_vector;
@ -3688,7 +3692,7 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
/* Check if vector type is a boolean vector. */
tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op))
&& vect_mask_constant_operand_p (stmt_vinfo, op_num))
&& vect_mask_constant_operand_p (vinfo, stmt_vinfo, op_num))
vector_type = truth_type_for (stmt_vectype);
else
vector_type = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), op_node);
@ -3797,8 +3801,8 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
constant_p = false;
if (TREE_CODE (orig_op) == SSA_NAME
&& !SSA_NAME_IS_DEFAULT_DEF (orig_op)
&& STMT_VINFO_BB_VINFO (stmt_vinfo)
&& (STMT_VINFO_BB_VINFO (stmt_vinfo)->bb
&& is_a <bb_vec_info> (vinfo)
&& (as_a <bb_vec_info> (vinfo)->bb
== gimple_bb (SSA_NAME_DEF_STMT (orig_op))))
place_after_defs = true;
@ -3823,12 +3827,12 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
stmt_vec_info last_stmt_info
= vect_find_last_scalar_stmt_in_slp (slp_node);
gsi = gsi_for_stmt (last_stmt_info->stmt);
init = vect_init_vector (stmt_vinfo, vec_cst, vector_type,
&gsi);
init = vect_init_vector (vinfo, stmt_vinfo, vec_cst,
vector_type, &gsi);
}
else
init = vect_init_vector (stmt_vinfo, vec_cst, vector_type,
NULL);
init = vect_init_vector (vinfo, stmt_vinfo, vec_cst,
vector_type, NULL);
if (ctor_seq != NULL)
{
gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (init));
@ -3902,7 +3906,8 @@ vect_get_slp_vect_defs (slp_tree slp_node, vec<tree> *vec_oprnds)
vect_get_slp_vect_defs () to retrieve them. */
void
vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
vect_get_slp_defs (vec_info *vinfo,
slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
{
if (n == -1U)
n = SLP_TREE_CHILDREN (slp_node).length ();
@ -3921,7 +3926,7 @@ vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
vect_get_slp_vect_defs (child, &vec_defs);
}
else
vect_get_constant_vectors (slp_node, i, &vec_defs);
vect_get_constant_vectors (vinfo, slp_node, i, &vec_defs);
vec_oprnds->quick_push (vec_defs);
}
@ -3933,13 +3938,13 @@ vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
SLP_NODE_INSTANCE. */
bool
vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
vect_transform_slp_perm_load (vec_info *vinfo,
slp_tree node, vec<tree> dr_chain,
gimple_stmt_iterator *gsi, poly_uint64 vf,
slp_instance slp_node_instance, bool analyze_only,
unsigned *n_perms)
{
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
vec_info *vinfo = stmt_info->vinfo;
int vec_index = 0;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
@ -4116,7 +4121,8 @@ vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
first_vec, second_vec,
mask_vec);
perm_stmt_info
= vect_finish_stmt_generation (stmt_info, perm_stmt,
= vect_finish_stmt_generation (vinfo,
stmt_info, perm_stmt,
gsi);
}
else
@ -4143,7 +4149,8 @@ vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
/* Vectorize SLP instance tree in postorder. */
static void
vect_schedule_slp_instance (slp_tree node, slp_instance instance)
vect_schedule_slp_instance (vec_info *vinfo,
slp_tree node, slp_instance instance)
{
gimple_stmt_iterator si;
stmt_vec_info stmt_info;
@ -4161,7 +4168,7 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
return;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
vect_schedule_slp_instance (child, instance);
vect_schedule_slp_instance (vinfo, child, instance);
/* Push SLP node def-type to stmts. */
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
@ -4219,11 +4226,11 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
vec<stmt_vec_info> v1;
unsigned j;
tree tmask = NULL_TREE;
vect_transform_stmt (stmt_info, &si, node, instance);
vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
v0 = SLP_TREE_VEC_STMTS (node).copy ();
SLP_TREE_VEC_STMTS (node).truncate (0);
gimple_assign_set_rhs_code (stmt, ocode);
vect_transform_stmt (stmt_info, &si, node, instance);
vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
gimple_assign_set_rhs_code (stmt, code0);
v1 = SLP_TREE_VEC_STMTS (node).copy ();
SLP_TREE_VEC_STMTS (node).truncate (0);
@ -4261,7 +4268,7 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
gimple_assign_lhs (v1[j]->stmt),
tmask);
SLP_TREE_VEC_STMTS (node).quick_push
(vect_finish_stmt_generation (stmt_info, vstmt, &si));
(vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si));
}
v0.release ();
v1.release ();
@ -4269,7 +4276,7 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
}
}
if (!done_p)
vect_transform_stmt (stmt_info, &si, node, instance);
vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
/* Restore stmt def-types. */
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
@ -4287,7 +4294,8 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
SLP instances may refer to the same scalar stmt. */
static void
vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
vect_remove_slp_scalar_calls (vec_info *vinfo,
slp_tree node, hash_set<slp_tree> &visited)
{
gimple *new_stmt;
gimple_stmt_iterator gsi;
@ -4303,7 +4311,7 @@ vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
return;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
vect_remove_slp_scalar_calls (child, visited);
vect_remove_slp_scalar_calls (vinfo, child, visited);
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
{
@ -4316,16 +4324,16 @@ vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
lhs = gimple_call_lhs (stmt);
new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
gsi = gsi_for_stmt (stmt);
stmt_info->vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
}
}
static void
vect_remove_slp_scalar_calls (slp_tree node)
vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
{
hash_set<slp_tree> visited;
vect_remove_slp_scalar_calls (node, visited);
vect_remove_slp_scalar_calls (vinfo, node, visited);
}
/* Vectorize the instance root. */
@ -4392,7 +4400,7 @@ vect_schedule_slp (vec_info *vinfo)
{
slp_tree node = SLP_INSTANCE_TREE (instance);
/* Schedule the tree of INSTANCE. */
vect_schedule_slp_instance (node, instance);
vect_schedule_slp_instance (vinfo, node, instance);
if (SLP_INSTANCE_ROOT_STMT (instance))
vectorize_slp_instance_root_stmt (node, instance);
@ -4416,7 +4424,7 @@ vect_schedule_slp (vec_info *vinfo)
stmts starting from the SLP tree root if they have no
uses. */
if (is_a <loop_vec_info> (vinfo))
vect_remove_slp_scalar_calls (root);
vect_remove_slp_scalar_calls (vinfo, root);
for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info)
&& j < SLP_INSTANCE_GROUP_SIZE (instance); j++)

982
gcc/tree-vect-stmts.c

File diff suppressed because it is too large Load Diff

1
gcc/tree-vectorizer.c

@ -631,7 +631,6 @@ stmt_vec_info
vec_info::new_stmt_vec_info (gimple *stmt)
{
stmt_vec_info res = XCNEW (class _stmt_vec_info);
res->vinfo = this;
res->stmt = stmt;
STMT_VINFO_TYPE (res) = undef_vec_info_type;

123
gcc/tree-vectorizer.h

@ -945,9 +945,6 @@ public:
/* The stmt to which this info struct refers to. */
gimple *stmt;
/* The vec_info with respect to which STMT is vectorized. */
vec_info *vinfo;
/* The vector type to be used for the LHS of this statement. */
tree vectype;
@ -1152,20 +1149,6 @@ struct gather_scatter_info {
/* Access Functions. */
#define STMT_VINFO_TYPE(S) (S)->type
#define STMT_VINFO_STMT(S) (S)->stmt
inline loop_vec_info
STMT_VINFO_LOOP_VINFO (stmt_vec_info stmt_vinfo)
{
if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (stmt_vinfo->vinfo))
return loop_vinfo;
return NULL;
}
inline bb_vec_info
STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
{
if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (stmt_vinfo->vinfo))
return bb_vinfo;
return NULL;
}
#define STMT_VINFO_RELEVANT(S) (S)->relevant
#define STMT_VINFO_LIVE_P(S) (S)->live
#define STMT_VINFO_VECTYPE(S) (S)->vectype
@ -1377,11 +1360,12 @@ extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt,
/* Alias targetm.vectorize.add_stmt_cost. */
static inline unsigned
add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
stmt_vec_info stmt_info, int misalign,
enum vect_cost_model_location where)
{
unsigned cost = targetm.vectorize.add_stmt_cost (data, count, kind,
unsigned cost = targetm.vectorize.add_stmt_cost (vinfo, data, count, kind,
stmt_info, misalign, where);
if (dump_file && (dump_flags & TDF_DETAILS))
dump_stmt_cost (dump_file, data, count, kind, stmt_info, misalign,
@ -1407,12 +1391,12 @@ destroy_cost_data (void *data)
}
inline void
add_stmt_costs (void *data, stmt_vector_for_cost *cost_vec)
add_stmt_costs (vec_info *vinfo, void *data, stmt_vector_for_cost *cost_vec)
{
stmt_info_for_cost *cost;
unsigned i;
FOR_EACH_VEC_ELT (*cost_vec, i, cost)
add_stmt_cost (data, cost->count, cost->kind, cost->stmt_info,
add_stmt_cost (vinfo, data, cost->count, cost->kind, cost->stmt_info,
cost->misalign, cost->where);
}
@ -1480,10 +1464,10 @@ vect_known_alignment_in_bytes (dr_vec_info *dr_info)
in DR_INFO itself). */
static inline innermost_loop_behavior *
vect_dr_behavior (dr_vec_info *dr_info)
vect_dr_behavior (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_info = dr_info->stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
if (loop_vinfo == NULL
|| !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
return &DR_INNERMOST (dr_info->dr);
@ -1496,11 +1480,12 @@ vect_dr_behavior (dr_vec_info *dr_info)
vect_dr_behavior to select the appropriate data_reference to use. */
inline tree
get_dr_vinfo_offset (dr_vec_info *dr_info, bool check_outer = false)
get_dr_vinfo_offset (vec_info *vinfo,
dr_vec_info *dr_info, bool check_outer = false)
{
innermost_loop_behavior *base;
if (check_outer)
base = vect_dr_behavior (dr_info);
base = vect_dr_behavior (vinfo, dr_info);
else
base = &dr_info->dr->innermost;
@ -1705,7 +1690,8 @@ extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
tree *, stmt_vec_info * = NULL,
gimple ** = NULL);
extern bool supportable_widening_operation (enum tree_code, stmt_vec_info,
extern bool supportable_widening_operation (vec_info *,
enum tree_code, stmt_vec_info,
tree, tree, enum tree_code *,
enum tree_code *, int *,
vec<tree> *);
@ -1715,31 +1701,36 @@ extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info,
int, enum vect_cost_model_location);
extern stmt_vec_info vect_finish_replace_stmt (stmt_vec_info, gimple *);
extern stmt_vec_info vect_finish_stmt_generation (stmt_vec_info, gimple *,
extern stmt_vec_info vect_finish_replace_stmt (vec_info *,
stmt_vec_info, gimple *);
extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
stmt_vec_info, gimple *,
gimple_stmt_iterator *);
extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
extern tree vect_get_store_rhs (stmt_vec_info);
extern tree vect_get_vec_def_for_operand_1 (stmt_vec_info, enum vect_def_type);
extern tree vect_get_vec_def_for_operand (tree, stmt_vec_info, tree = NULL);
extern void vect_get_vec_defs (tree, tree, stmt_vec_info, vec<tree> *,
vec<tree> *, slp_tree);
extern tree vect_get_vec_def_for_operand (vec_info *, tree,
stmt_vec_info, tree = NULL);
extern void vect_get_vec_defs (vec_info *, tree, tree, stmt_vec_info,
vec<tree> *, vec<tree> *, slp_tree);
extern void vect_get_vec_defs_for_stmt_copy (vec_info *,
vec<tree> *, vec<tree> *);
extern tree vect_init_vector (stmt_vec_info, tree, tree,
extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
gimple_stmt_iterator *);
extern tree vect_get_vec_def_for_stmt_copy (vec_info *, tree);
extern bool vect_transform_stmt (stmt_vec_info, gimple_stmt_iterator *,
extern bool vect_transform_stmt (vec_info *, stmt_vec_info,
gimple_stmt_iterator *,
slp_tree, slp_instance);
extern void vect_remove_stores (stmt_vec_info);
extern void vect_remove_stores (vec_info *, stmt_vec_info);
extern bool vect_nop_conversion_p (stmt_vec_info);
extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree,
extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
slp_tree,
slp_instance, stmt_vector_for_cost *);
extern void vect_get_load_cost (stmt_vec_info, int, bool,
extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, bool,
unsigned int *, unsigned int *,
stmt_vector_for_cost *,
stmt_vector_for_cost *, bool);
extern void vect_get_store_cost (stmt_vec_info, int,
extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
unsigned int *, stmt_vector_for_cost *);
extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);
@ -1747,22 +1738,24 @@ extern tree vect_gen_perm_mask_checked (tree, const vec_perm_indices &);
extern void optimize_mask_stores (class loop*);
extern gcall *vect_gen_while (tree, tree, tree);
extern tree vect_gen_while_not (gimple_seq *, tree, tree, tree);
extern opt_result vect_get_vector_types_for_stmt (stmt_vec_info, tree *,
extern opt_result vect_get_vector_types_for_stmt (vec_info *,
stmt_vec_info, tree *,
tree *, unsigned int = 0);
extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
extern enum dr_alignment_support vect_supportable_dr_alignment
(dr_vec_info *, bool);
(vec_info *, dr_vec_info *, bool);
extern tree vect_get_smallest_scalar_type (stmt_vec_info, HOST_WIDE_INT *,
HOST_WIDE_INT *);
extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
extern bool vect_slp_analyze_instance_dependence (slp_instance);
extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
extern opt_result vect_enhance_data_refs_alignment (loop_vec_info);
extern opt_result vect_analyze_data_refs_alignment (loop_vec_info);
extern opt_result vect_verify_datarefs_alignment (loop_vec_info);
extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance);
extern bool vect_slp_analyze_and_verify_instance_alignment (vec_info *,
slp_instance);
extern opt_result vect_analyze_data_ref_accesses (vec_info *);
extern opt_result vect_prune_runtime_alias_test_list (loop_vec_info);
extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
@ -1773,11 +1766,12 @@ extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
vec<data_reference_p> *);
extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *);
extern void vect_record_base_alignments (vec_info *);
extern tree vect_create_data_ref_ptr (stmt_vec_info, tree, class loop *, tree,
extern tree vect_create_data_ref_ptr (vec_info *,
stmt_vec_info, tree, class loop *, tree,
tree *, gimple_stmt_iterator *,
gimple **, bool,
tree = NULL_TREE, tree = NULL_TREE);
extern tree bump_vector_ptr (tree, gimple *, gimple_stmt_iterator *,
extern tree bump_vector_ptr (vec_info *, tree, gimple *, gimple_stmt_iterator *,
stmt_vec_info, tree);
extern void vect_copy_ref_info (tree, tree);
extern tree vect_create_destination_var (tree, tree);
@ -1785,18 +1779,22 @@ extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
extern void vect_permute_store_chain (vec<tree> ,unsigned int, stmt_vec_info,
gimple_stmt_iterator *, vec<tree> *);
extern tree vect_setup_realignment (stmt_vec_info, gimple_stmt_iterator *,
extern void vect_permute_store_chain (vec_info *,
vec<tree> ,unsigned int, stmt_vec_info,
gimple_stmt_iterator *, vec<tree> *);
extern tree vect_setup_realignment (vec_info *,
stmt_vec_info, gimple_stmt_iterator *,
tree *, enum dr_alignment_support, tree,
class loop **);
extern void vect_transform_grouped_load (stmt_vec_info, vec<tree> , int,
gimple_stmt_iterator *);
extern void vect_record_grouped_load_vectors (stmt_vec_info, vec<tree>);
extern void vect_transform_grouped_load (vec_info *, stmt_vec_info, vec<tree>,
int, gimple_stmt_iterator *);
extern void vect_record_grouped_load_vectors (vec_info *,
stmt_vec_info, vec<tree>);
extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
const char * = NULL);
extern tree vect_create_addr_base_for_vector_ref (stmt_vec_info, gimple_seq *,
extern tree vect_create_addr_base_for_vector_ref (vec_info *,
stmt_vec_info, gimple_seq *,
tree, tree = NULL_TREE);
/* In tree-vect-loop.c. */
@ -1818,25 +1816,31 @@ extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
unsigned int, tree, tree);
extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
unsigned int, tree, unsigned int);
extern stmt_vec_info info_for_reduction (stmt_vec_info);
extern stmt_vec_info info_for_reduction (vec_info *, stmt_vec_info);
/* Drive for loop transformation stage. */
extern class loop *vect_transform_loop (loop_vec_info, gimple *);
extern opt_loop_vec_info vect_analyze_loop_form (class loop *,
vec_info_shared *);
extern bool vectorizable_live_operation (stmt_vec_info, gimple_stmt_iterator *,
extern bool vectorizable_live_operation (loop_vec_info,
stmt_vec_info, gimple_stmt_iterator *,
slp_tree, slp_instance, int,
bool, stmt_vector_for_cost *);
extern bool vectorizable_reduction (stmt_vec_info, slp_tree, slp_instance,
extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
slp_tree, slp_instance,
stmt_vector_for_cost *);
extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *,
extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
gimple_stmt_iterator *,
stmt_vec_info *, slp_tree,
stmt_vector_for_cost *);
extern bool vect_transform_reduction (stmt_vec_info, gimple_stmt_iterator *,
extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
gimple_stmt_iterator *,
stmt_vec_info *, slp_tree);
extern bool vect_transform_cycle_phi (stmt_vec_info, stmt_vec_info *,
extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
stmt_vec_info *,
slp_tree, slp_instance);
extern bool vectorizable_lc_phi (stmt_vec_info, stmt_vec_info *, slp_tree);
extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info,
stmt_vec_info *, slp_tree);
extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code);
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
stmt_vector_for_cost *,
@ -1846,7 +1850,7 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
/* In tree-vect-slp.c. */
extern void vect_free_slp_instance (slp_instance, bool);
extern bool vect_transform_slp_perm_load (slp_tree, vec<tree> ,
extern bool vect_transform_slp_perm_load (vec_info *, slp_tree, vec<tree>,
gimple_stmt_iterator *, poly_uint64,
slp_instance, bool, unsigned *);
extern bool vect_slp_analyze_operations (vec_info *);
@ -1854,7 +1858,8 @@ extern void vect_schedule_slp (vec_info *);
extern opt_result vect_analyze_slp (vec_info *, unsigned);
extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_get_slp_defs (slp_tree, vec<vec<tree> > *, unsigned n = -1U);
extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
unsigned n = -1U);
extern bool vect_slp_bb (basic_block);
extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree);
extern bool is_simple_and_all_uses_invariant (stmt_vec_info, loop_vec_info);