Skip to content

Commit

Permalink
8344035: Replace predicate walking code in Loop Unswitching with a pr…
Browse files Browse the repository at this point in the history 
…edicate visitor
  • Loading branch information
@chhagedorn
chhagedorn committed Nov 28, 2024
1 parent e689a36 commit 310c285
Show file tree
Hide file tree
Showing 6 changed files with 212 additions and 205 deletions.
145 changes: 5 additions & 140 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,150 +272,15 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_
orig_to_clone.iterate_all(orig_clone_action);
}

IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
Node* new_entry, Deoptimization::DeoptReason reason,
const bool slow_loop) {

IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
slow_loop);
assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
return new_predicate_proj;
}

// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its
// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates
// to the new cloned predicates.
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateNode* false_path_loop_parse_predicate) {
// Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the
// original predicate order.
Unique_Node_List list;
get_template_assertion_predicates(old_parse_predicate_proj, list);

Node_List to_process;
for (int i = list.size() - 1; i >= 0; i--) {
IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue();
assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node");

IfTrueNode* true_path_loop_proj =
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
true_path_loop_parse_predicate);
IfTrueNode* false_path_loop_proj =
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
false_path_loop_parse_predicate);

// Update control dependent data nodes.
for (DUIterator j = template_assertion_predicate_success_proj->outs();
template_assertion_predicate_success_proj->has_out(j);
j++) {
Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j);
if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) {
assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
Node* false_path_loop_node = old_new[true_path_loop_node->_idx];
assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
_igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj);
to_process.push(false_path_loop_node);
--j;
}
}
// Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them.
while (to_process.size() > 0) {
Node* slow_node = to_process.pop();
_igvn.replace_input_of(slow_node, 0, false_path_loop_proj);
}
}
}

// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead
// of the projections.
void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list,
const bool get_opaque) {
// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree.
void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
Unique_Node_List& list) {
Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason();
PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason);
TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque);
predicate_iterator.for_each(template_assertion_predicate_collector);
}

// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
// predicate again).
IfTrueNode*
PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
ParsePredicateNode* unswitched_loop_parse_predicate) {
TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj);
IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this);
_igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj);
set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj));
return template_success_proj;
}

// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the
// unswitch If.
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry) {
LoopNode* head = loop->_head->as_Loop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);

const Predicates predicates(entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(),
Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(),
Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry);

const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) {
// Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
// is only created when converting a LoopNode to a CountedLoopNode).
clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check,
true_path_loop_entry, false_path_loop_entry);
}
}

// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block.
void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry) {
if (predicate_block->has_parse_predicate()) {
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
// be changed with the complete fix for Assertion Predicates.
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry);
assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(),
"must be success projections of the cloned Parse Predicates");
clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(),
true_path_loop_entry->in(0)->as_ParsePredicate(),
false_path_loop_entry->in(0)->as_ParsePredicate());
}
OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list);
predicate_iterator.for_each(opaque_template_assertion_predicate_collector);
}

void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
assert(predicate_block->has_parse_predicate(), "must have parse predicate");
ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj();
iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false);
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);

ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true);
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
}

#ifndef PRODUCT
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) {
assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow");
new_entry->in(0)->dump();
}
}
#endif

//------------------------------Invariance-----------------------------------
// Helper class for loop_predication_impl to compute invariance on the fly and
// clone invariants.
Expand Down
67 changes: 39 additions & 28 deletions src/hotspot/share/opto/loopUnswitch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,18 +236,47 @@ class OriginalLoop : public StackObj {
_phase(loop->_phase) {}
NONCOPYABLE(OriginalLoop);

// Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
// Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
const uint first_false_path_loop_node_index = _phase->C->unique();
clone_loop(unswitched_loop_selector);

move_parse_and_template_assertion_predicates_to_unswitched_loops(first_false_path_loop_node_index);
DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)

_phase->recompute_dom_depth();
remove_unswitch_candidate_from_loops(unswitched_loop_selector);
}

private:
void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) {
_phase->replace_loop_entry(_loop_head, true_path_loop_entry);
void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) {
_phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
fix_loop_entries(unswitched_loop_selector);
}

void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) {
_phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj());
LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
_phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry);
_phase->replace_loop_entry(false_path_loop_strip_mined_head,
unswitched_loop_selector.false_path_loop_proj());
}

Node* old_to_new(const Node* old) const {
return _old_new[old->_idx];
// Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the
// loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before
// the unswitched loop selector are killed.
void move_parse_and_template_assertion_predicates_to_unswitched_loops(const uint first_false_path_loop_node_index) const {
const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new);
const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index);
CloneUnswitchedLoopPredicatesVisitor
clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body,
node_in_false_path_loop_body, _phase);
PredicateIterator predicate_iterator(_loop_head->in(LoopNode::EntryControl));
predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor);
}

#ifdef ASSERT
#ifdef ASSERT
void verify_unswitched_loop_versions(LoopNode* true_path_loop_head,
const UnswitchedLoopSelector& unswitched_loop_selector) const {
verify_unswitched_loop_version(true_path_loop_head, unswitched_loop_selector.true_path_loop_proj());
Expand All @@ -263,6 +292,10 @@ class OriginalLoop : public StackObj {
}
#endif // ASSERT

Node* old_to_new(const Node* old) const {
return _old_new[old->_idx];
}

// Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector
// If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting
// the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round.
Expand All @@ -276,28 +309,6 @@ class OriginalLoop : public StackObj {
_phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone);
}

public:
// Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
// Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
_phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());

// At this point, the selector If projections are the corresponding loop entries.
// clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector
// If projections. The loop entries are updated accordingly.
IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj();
IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj();
_phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new,
true_path_loop_entry, false_path_loop_entry);

fix_loop_entries(true_path_loop_entry, false_path_loop_entry);

DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)

_phase->recompute_dom_depth();
remove_unswitch_candidate_from_loops(unswitched_loop_selector);
}
};

// See comments below file header for more information about Loop Unswitching.
Expand Down
4 changes: 2 additions & 2 deletions src/hotspot/share/opto/loopnode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4470,15 +4470,15 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
if (profiled_loop_predicate_block->has_parse_predicate()) {
ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
}
}

if (UseLoopPredicate) {
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
if (loop_predicate_block->has_parse_predicate()) {
ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
}
}
}
Expand Down
24 changes: 2 additions & 22 deletions src/hotspot/share/opto/loopnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -942,7 +942,8 @@ class PhaseIdealLoop : public PhaseTransform {
static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
#endif
private:
static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false);
static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
Unique_Node_List& list);
void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head);
void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
CountedLoopNode* remaining_loop_head,
Expand Down Expand Up @@ -1657,28 +1658,7 @@ class PhaseIdealLoop : public PhaseTransform {
_nodes_required = UINT_MAX;
}

public:
// Clone Parse Predicates to slow and fast loop when unswitching a loop
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
private:
void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop);
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateNode* false_path_loop_parse_predicate);
IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
ParsePredicateNode* unswitched_loop_parse_predicate);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;

bool _created_loop_node;
DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)
Expand Down
Loading

0 comments on commit 310c285

Please sign in to comment.