mirror/gcc
2
0
Fork 0
mirror of git://gcc.gnu.org/git/gcc.git synced 2025-04-03 05:10:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

re PR tree-optimization/33319 (g++.dg/tree-ssa/pr27549.C ICE with vectorization)

Browse Source 
PR tree-optimization/33319
        * tree-vect-analyze.c (vect_same_range_drs): New.
        (vect_vfa_range_equal): New.
        (vect_is_duplicate_ddr): Removed.
        (vect_mark_for_runtime_alias_test): Do not perform marking when
        optimizing for size or max_param for alias checking is zero.
        Move the function before vect_analyze_data_ref_dependence.
        (vect_analyze_data_ref_dependence): Add call to
        vect_mark_for_runtime_alias_test in two cases when dependence
        is not clear.
        (vect_analyze_data_ref_dependences): Do not call to
        vect_mark_for_runtime_alias_test.
        (vect_prune_runtime_alias_test_list): New.
        (vect_analyze_loop): Add call to vect_prune_runtime_alias_test_list.
        * tree-vect-transform.c (vect_estimate_min_profitable_iters):
        Update vec_outside_cost.
        (vect_vfa_segment_size): More compact code, use TYPE_SIZE_UNIT.
        (vect_create_cond_for_alias_checks): Build the base address of data
        reference from DR_GROUP_FIRST_DR.
        (vect_loop_versioning): New.
        (vect_transform_loop): Add a call to vect_loop_versioning.
        Remove factored out code.

From-SVN: r128539
This commit is contained in:
Victor Kaplansky 2007-09-17 09:37:31 +00:00 committed by Victor Kaplansky
parent 2adde4ffdc
commit 42cbdeac68
3 changed files with 335 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
gcc/ChangeLog
View File

@ -1,3 +1,28 @@
2007-09-17 Victor Kaplansky <victork@il.ibm.com>
PR tree-optimization/33319
* tree-vect-analyze.c (vect_same_range_drs): New.
(vect_vfa_range_equal): New.
(vect_is_duplicate_ddr): Removed.
(vect_mark_for_runtime_alias_test): Do not perform marking when
optimizing for size or max_param for alias checking is zero.
Move the function before vect_analyze_data_ref_dependence.
(vect_analyze_data_ref_dependence): Add call to
vect_mark_for_runtime_alias_test in two cases when dependence
is not clear.
(vect_analyze_data_ref_dependences): Do not call to
vect_mark_for_runtime_alias_test.
(vect_prune_runtime_alias_test_list): New.
(vect_analyze_loop): Add call to vect_prune_runtime_alias_test_list.
* tree-vect-transform.c (vect_estimate_min_profitable_iters):
Update vec_outside_cost.
(vect_vfa_segment_size): More compact code, use TYPE_SIZE_UNIT.
(vect_create_cond_for_alias_checks): Build the base address of data
reference from DR_GROUP_FIRST_DR.
(vect_loop_versioning): New.
(vect_transform_loop): Add a call to vect_loop_versioning.
Remove factored out code.
2007-09-16 John David Anglin <dave.anglin@nrc-cnrc.gc.ca>
PR middle-end/33273

256
gcc/tree-vect-analyze.c
View File

@ -1118,11 +1118,85 @@ vect_check_interleaving (struct data_reference *dra,
}
}
/* Check if data references pointed by DR_I and DR_J are same or
belong to same interleaving group. Return FALSE if drs are
different, otherwise return TRUE. */
static bool
vect_same_range_drs (data_reference_p dr_i, data_reference_p dr_j)
{
tree stmt_i = DR_STMT (dr_i);
tree stmt_j = DR_STMT (dr_j);
if (operand_equal_p (DR_REF (dr_i), DR_REF (dr_j), 0)
|| (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i))
&& DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j))
&& (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i))
== DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j)))))
return true;
else
return false;
}
/* If address ranges represented by DDR_I and DDR_J are equal,
return TRUE, otherwise return FALSE. */
static bool
vect_vfa_range_equal (ddr_p ddr_i, ddr_p ddr_j)
{
if ((vect_same_range_drs (DDR_A (ddr_i), DDR_A (ddr_j))
&& vect_same_range_drs (DDR_B (ddr_i), DDR_B (ddr_j)))
|| (vect_same_range_drs (DDR_A (ddr_i), DDR_B (ddr_j))
&& vect_same_range_drs (DDR_B (ddr_i), DDR_A (ddr_j))))
return true;
else
return false;
}
/* Insert DDR into LOOP_VINFO list of ddrs that may alias and need to be
tested at run-time. Return TRUE if DDR was successfully inserted.
Return false if versioning is not supported. */
static bool
vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
if ((unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS) == 0)
return false;
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump, "mark for run-time aliasing test between ");
print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM);
}
if (optimize_size)
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
fprintf (vect_dump, "versioning not supported when optimizing for size.");
return false;
}
/* FORNOW: We don't support versioning with outer-loop vectorization. */
if (loop->inner)
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
fprintf (vect_dump, "versioning not yet supported for outer-loops.");
return false;
}
VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr);
return true;
}
/* Function vect_analyze_data_ref_dependence.
Return TRUE if there (might) exist a dependence between a memory-reference
DRA and a memory-reference DRB. */
DRA and a memory-reference DRB. When versioning for alias may check a
dependence at run-time, return FALSE. */
static bool
vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
@ -1160,7 +1234,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
return true;
/* Add to list of ddrs that need to be tested at run-time. */
return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
}
if (DDR_NUM_DIST_VECTS (ddr) == 0)
@ -1172,7 +1247,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
return true;
/* Add to list of ddrs that need to be tested at run-time. */
return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
}
loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
@ -1224,10 +1300,10 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
continue;
}
if (vect_print_dump_info (REPORT_DR_DETAILS))
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
{
fprintf (vect_dump,
"versioning for alias required: possible dependence "
"not vectorized, possible dependence "
"between data-refs ");
print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
fprintf (vect_dump, " and ");
@ -1240,88 +1316,6 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
return false;
}
/* Return TRUE if DDR_NEW is already found in MAY_ALIAS_DDRS list. */
static bool
vect_is_duplicate_ddr (VEC (ddr_p, heap) * may_alias_ddrs, ddr_p ddr_new)
{
unsigned i;
ddr_p ddr;
for (i = 0; VEC_iterate (ddr_p, may_alias_ddrs, i, ddr); i++)
{
tree dref_A_i, dref_B_i, dref_A_j, dref_B_j;
dref_A_i = DR_REF (DDR_A (ddr));
dref_B_i = DR_REF (DDR_B (ddr));
dref_A_j = DR_REF (DDR_A (ddr_new));
dref_B_j = DR_REF (DDR_B (ddr_new));
if ((operand_equal_p (dref_A_i, dref_A_j, 0)
&& operand_equal_p (dref_B_i, dref_B_j, 0))
|| (operand_equal_p (dref_A_i, dref_B_j, 0)
&& operand_equal_p (dref_B_i, dref_A_j, 0)))
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump, "found same pair of data references ");
print_generic_expr (vect_dump, dref_A_i, TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, dref_B_i, TDF_SLIM);
}
return true;
}
}
return false;
}
/* Save DDR in LOOP_VINFO list of ddrs that may alias and need to be
tested at run-time. Returns false if number of run-time checks
inserted by vectorizer is greater than maximum defined by
PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS. */
static bool
vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump, "mark for run-time aliasing test between ");
print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM);
}
/* FORNOW: We don't support versioning with outer-loop vectorization. */
if (loop->inner)
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
fprintf (vect_dump, "versioning not yet supported for outer-loops.");
return false;
}
/* Do not add to the list duplicate ddrs. */
if (vect_is_duplicate_ddr (LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr))
return true;
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))
>= (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS))
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump,
"disable versioning for alias - max number of generated "
"checks exceeded.");
}
VEC_truncate (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), 0);
return false;
}
VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr);
return true;
}
/* Function vect_analyze_data_ref_dependences.
Examine all the data references in the loop, and make sure there do not
@ -1339,11 +1333,7 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo)
for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
if (vect_analyze_data_ref_dependence (ddr, loop_vinfo))
{
/* Add to list of ddrs that need to be tested at run-time. */
if (!vect_mark_for_runtime_alias_test (ddr, loop_vinfo))
return false;
}
return true;
}
@ -2381,6 +2371,77 @@ vect_analyze_data_ref_accesses (loop_vec_info loop_vinfo)
return true;
}
/* Function vect_prune_runtime_alias_test_list.
Prune a list of ddrs to be tested at run-time by versioning for alias.
Return FALSE if resulting list of ddrs is longer then allowed by
PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE. */
static bool
vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
{
VEC (ddr_p, heap) * ddrs =
LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo);
unsigned i, j;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_prune_runtime_alias_test_list ===");
for (i = 0; i < VEC_length (ddr_p, ddrs); )
{
bool found;
ddr_p ddr_i;
ddr_i = VEC_index (ddr_p, ddrs, i);
found = false;
for (j = 0; j < i; j++)
{
ddr_p ddr_j = VEC_index (ddr_p, ddrs, j);
if (vect_vfa_range_equal (ddr_i, ddr_j))
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump, "found equal ranges ");
print_generic_expr (vect_dump, DR_REF (DDR_A (ddr_i)), TDF_SLIM);
fprintf (vect_dump, ", ");
print_generic_expr (vect_dump, DR_REF (DDR_B (ddr_i)), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (DDR_A (ddr_j)), TDF_SLIM);
fprintf (vect_dump, ", ");
print_generic_expr (vect_dump, DR_REF (DDR_B (ddr_j)), TDF_SLIM);
}
found = true;
break;
}
}
if (found)
{
VEC_ordered_remove (ddr_p, ddrs, i);
continue;
}
i++;
}
if (VEC_length (ddr_p, ddrs) >
(unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS))
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump,
"disable versioning for alias - max number of generated "
"checks exceeded.");
}
VEC_truncate (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), 0);
return false;
}
return true;
}
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
@ -4231,6 +4292,19 @@ vect_analyze_loop (struct loop *loop)
return NULL;
}
/* Prune the list of ddrs to be tested at run-time by versioning for alias.
It is important to call pruning after vect_analyze_data_ref_accesses,
since we use grouping information gathered by interleaving analysis. */
ok = vect_prune_runtime_alias_test_list (loop_vinfo);
if (!ok)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "too long list of versioning for alias "
"run-time tests.");
destroy_loop_vec_info (loop_vinfo, true);
return NULL;
}
/* Check the SLP opportunities in the loop, analyze and build SLP trees. */
ok = vect_analyze_slp (loop_vinfo);
if (ok)

254
gcc/tree-vect-transform.c
View File

@ -137,15 +137,32 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo)
return 0;
}
/* Requires loop versioning tests to handle misalignment.
FIXME: Make cost depend on number of stmts in may_misalign list. */
/* Requires loop versioning tests to handle misalignment. */
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
{
vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST;
/* FIXME: Make cost depend on complexity of individual check. */
vec_outside_cost +=
VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
"versioning.\n");
"versioning to treat misalignment.\n");
}
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
{
/* FIXME: Make cost depend on complexity of individual check. */
vec_outside_cost +=
VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
"versioning aliasing.\n");
}
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|| VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
{
vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST;
}
/* Count statements in scalar loop. Using this as scalar cost for a single
@ -6864,31 +6881,18 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo,
static tree
vect_vfa_segment_size (struct data_reference *dr, tree vect_factor)
{
tree segment_length;
tree segment_length = fold_build2 (MULT_EXPR, integer_type_node,
DR_STEP (dr), vect_factor);
if (vect_supportable_dr_alignment (dr) == dr_explicit_realign_optimized)
{
tree vector_size =
build_int_cst (integer_type_node,
GET_MODE_SIZE (TYPE_MODE (STMT_VINFO_VECTYPE
(vinfo_for_stmt (DR_STMT (dr))))));
tree vector_size = TYPE_SIZE_UNIT
(STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr))));
segment_length =
fold_convert (sizetype,
fold_build2 (PLUS_EXPR, integer_type_node,
fold_build2 (MULT_EXPR, integer_type_node, DR_STEP (dr),
vect_factor),
vector_size));
segment_length = fold_build2 (PLUS_EXPR, integer_type_node,
segment_length, vector_size);
}
else
{
segment_length =
fold_convert (sizetype,
fold_build2 (MULT_EXPR, integer_type_node, DR_STEP (dr),
vect_factor));
}
return segment_length;
return fold_convert (sizetype, segment_length);
}
/* Function vect_create_cond_for_alias_checks.
@ -6907,6 +6911,8 @@ vect_vfa_segment_size (struct data_reference *dr, tree vect_factor)
COND_EXPR - conditional expression.
COND_EXPR_STMT_LIST - statements needed to construct the conditional
expression.
The returned value is the conditional expression to be used in the if
statement that controls which version of the loop gets executed at runtime.
*/
@ -6940,26 +6946,47 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo,
for (i = 0; VEC_iterate (ddr_p, may_alias_ddrs, i, ddr); i++)
{
tree stmt_a = DR_STMT (DDR_A (ddr));
tree stmt_b = DR_STMT (DDR_B (ddr));
struct data_reference *dr_a, *dr_b;
tree dr_group_first_a, dr_group_first_b;
tree addr_base_a, addr_base_b;
tree segment_length_a, segment_length_b;
tree stmt_a, stmt_b;
tree addr_base_a =
dr_a = DDR_A (ddr);
stmt_a = DR_STMT (DDR_A (ddr));
dr_group_first_a = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_a));
if (dr_group_first_a)
{
stmt_a = dr_group_first_a;
dr_a = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_a));
}
dr_b = DDR_B (ddr);
stmt_b = DR_STMT (DDR_B (ddr));
dr_group_first_b = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_b));
if (dr_group_first_b)
{
stmt_b = dr_group_first_b;
dr_b = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_b));
}
addr_base_a =
vect_create_addr_base_for_vector_ref (stmt_a, cond_expr_stmt_list,
NULL_TREE, loop);
tree addr_base_b =
addr_base_b =
vect_create_addr_base_for_vector_ref (stmt_b, cond_expr_stmt_list,
NULL_TREE, loop);
tree segment_length_a = vect_vfa_segment_size (DDR_A (ddr), vect_factor);
tree segment_length_b = vect_vfa_segment_size (DDR_B (ddr), vect_factor);
segment_length_a = vect_vfa_segment_size (dr_a, vect_factor);
segment_length_b = vect_vfa_segment_size (dr_b, vect_factor);
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
fprintf (vect_dump,
"create runtime check for data references ");
print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM);
print_generic_expr (vect_dump, DR_REF (dr_a), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM);
print_generic_expr (vect_dump, DR_REF (dr_b), TDF_SLIM);
}
@ -6988,6 +7015,91 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo,
}
/* Function vect_loop_versioning.
If the loop has data references that may or may not be aligned or/and
has data reference relations whose independence was not proven then
two versions of the loop need to be generated, one which is vectorized
and one which isn't. A test is then generated to control which of the
loops is executed. The test checks for the alignment of all of the
data references that may or may not be aligned. An additional
sequence of runtime tests is generated for each pairs of DDRs whose
independence was not proven. The vectorized version of loop is
executed only if both alias and alignment tests are passed. */
static void
vect_loop_versioning (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct loop *nloop;
tree cond_expr = NULL_TREE;
tree cond_expr_stmt_list = NULL_TREE;
basic_block condition_bb;
block_stmt_iterator cond_exp_bsi;
basic_block merge_bb;
basic_block new_exit_bb;
edge new_exit_e, e;
tree orig_phi, new_phi, arg;
unsigned prob = 4 * REG_BR_PROB_BASE / 5;
tree gimplify_stmt_list;
if (!VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
&& !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
return;
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
cond_expr =
vect_create_cond_for_align_checks (loop_vinfo, &cond_expr_stmt_list);
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr, &cond_expr_stmt_list);
cond_expr =
fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node);
cond_expr =
force_gimple_operand (cond_expr, &gimplify_stmt_list, true,
NULL_TREE);
append_to_statement_list (gimplify_stmt_list, &cond_expr_stmt_list);
initialize_original_copy_tables ();
nloop = loop_version (loop, cond_expr, &condition_bb,
prob, prob, REG_BR_PROB_BASE - prob, true);
free_original_copy_tables();
/* Loop versioning violates an assumption we try to maintain during
vectorization - that the loop exit block has a single predecessor.
After versioning, the exit block of both loop versions is the same
basic block (i.e. it has two predecessors). Just in order to simplify
following transformations in the vectorizer, we fix this situation
here by adding a new (empty) block on the exit-edge of the loop,
with the proper loop-exit phis to maintain loop-closed-form. */
merge_bb = single_exit (loop)->dest;
gcc_assert (EDGE_COUNT (merge_bb->preds) == 2);
new_exit_bb = split_edge (single_exit (loop));
new_exit_e = single_exit (loop);
e = EDGE_SUCC (new_exit_bb, 0);
for (orig_phi = phi_nodes (merge_bb); orig_phi;
orig_phi = PHI_CHAIN (orig_phi))
{
new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),
new_exit_bb);
arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
add_phi_arg (new_phi, arg, new_exit_e);
SET_PHI_ARG_DEF (orig_phi, e->dest_idx, PHI_RESULT (new_phi));
}
/* End loop-exit-fixes after versioning. */
update_ssa (TODO_update_ssa);
if (cond_expr_stmt_list)
{
cond_exp_bsi = bsi_last (condition_bb);
bsi_insert_before (&cond_exp_bsi, cond_expr_stmt_list, BSI_SAME_STMT);
}
}
/* Remove a group of stores (for SLP or interleaving), free their
stmt_vec_info. */
@ -7096,7 +7208,6 @@ vect_schedule_slp (loop_vec_info loop_vinfo, unsigned int nunits)
return is_store;
}
/* Function vect_transform_loop.
The analysis phase has determined that the loop is vectorizable.
@ -7119,82 +7230,7 @@ vect_transform_loop (loop_vec_info loop_vinfo)
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vec_transform_loop ===");
/* If the loop has data references that may or may not be aligned or/and
has data reference relations whose independence was not proven then
two versions of the loop need to be generated, one which is vectorized
and one which isn't. A test is then generated to control which of the
loops is executed. The test checks for the alignment of all of the
data references that may or may not be aligned. An additional
sequence of runtime tests is generated for each pairs of DDRs whose
independence was not proven. The vectorized version of loop is
executed only if both alias and alignment tests are passed. */
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|| VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
{
struct loop *nloop;
tree cond_expr = NULL_TREE;
tree cond_expr_stmt_list = NULL_TREE;
basic_block condition_bb;
block_stmt_iterator cond_exp_bsi;
basic_block merge_bb;
basic_block new_exit_bb;
edge new_exit_e, e;
tree orig_phi, new_phi, arg;
unsigned prob = 4 * REG_BR_PROB_BASE / 5;
tree gimplify_stmt_list;
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
cond_expr =
vect_create_cond_for_align_checks (loop_vinfo, &cond_expr_stmt_list);
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr,
&cond_expr_stmt_list);
cond_expr =
fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node);
cond_expr =
force_gimple_operand (cond_expr, &gimplify_stmt_list, true,
NULL_TREE);
append_to_statement_list (gimplify_stmt_list, &cond_expr_stmt_list);
initialize_original_copy_tables ();
nloop = loop_version (loop, cond_expr, &condition_bb,
prob, prob, REG_BR_PROB_BASE - prob, true);
free_original_copy_tables();
/** Loop versioning violates an assumption we try to maintain during
vectorization - that the loop exit block has a single predecessor.
After versioning, the exit block of both loop versions is the same
basic block (i.e. it has two predecessors). Just in order to simplify
following transformations in the vectorizer, we fix this situation
here by adding a new (empty) block on the exit-edge of the loop,
with the proper loop-exit phis to maintain loop-closed-form. **/
merge_bb = single_exit (loop)->dest;
gcc_assert (EDGE_COUNT (merge_bb->preds) == 2);
new_exit_bb = split_edge (single_exit (loop));
new_exit_e = single_exit (loop);
e = EDGE_SUCC (new_exit_bb, 0);
for (orig_phi = phi_nodes (merge_bb); orig_phi;
orig_phi = PHI_CHAIN (orig_phi))
{
new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),
new_exit_bb);
arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
add_phi_arg (new_phi, arg, new_exit_e);
SET_PHI_ARG_DEF (orig_phi, e->dest_idx, PHI_RESULT (new_phi));
}
/** end loop-exit-fixes after versioning **/
update_ssa (TODO_update_ssa);
cond_exp_bsi = bsi_last (condition_bb);
bsi_insert_before (&cond_exp_bsi, cond_expr_stmt_list, BSI_SAME_STMT);
}
vect_loop_versioning (loop_vinfo);
/* CHECKME: we wouldn't need this if we called update_ssa once
for all loops. */