Skip to content

Commit

Permalink
8341977: Replace predicate walking and cloning code for Loop Peeling …
Browse files Browse the repository at this point in the history
…with a predicate visitor
  • Loading branch information
chhagedorn committed Oct 24, 2024
1 parent 7131f05 commit aeca241
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 66 deletions.
90 changes: 28 additions & 62 deletions src/hotspot/share/opto/loopTransform.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -762,7 +762,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {

// Step 1: Clone the loop body. The clone becomes the peeled iteration.
// The pre-loop illegally has 2 control users (old & new loops).
const uint idx_before_clone = Compile::current()->unique();
const uint first_node_index_in_cloned_loop_body = Compile::current()->unique();
LoopNode* outer_loop_head = head->skip_strip_mined();
clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined);

Expand Down Expand Up @@ -815,19 +815,8 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {

// Step 5: Assertion Predicates initialization
if (counted_loop && UseLoopPredicate) {
CountedLoopNode *cl_head = head->as_CountedLoop();
Node* init = cl_head->init_trip();
Node* stride = cl_head->stride();
IdealLoopTree* outer_loop = get_loop(outer_loop_head);
const Predicates predicates(new_head->in(LoopNode::EntryControl));
initialize_assertion_predicates_for_peeled_loop(predicates.loop_predicate_block(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_assertion_predicates_for_peeled_loop(predicates.profiled_loop_predicate_block(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_assertion_predicates_for_peeled_loop(new_head->as_CountedLoop(), head->as_CountedLoop(),
first_node_index_in_cloned_loop_body, old_new);
}

// Now force out all loop-invariant dominating tests. The optimizer
Expand Down Expand Up @@ -1434,7 +1423,6 @@ bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
assert(stride == 0 || init != 0, "init should be there every time stride is");
return init != 0;
}
#endif // ASSERT

void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) {
init = 0;
Expand All @@ -1460,6 +1448,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
}
}
}
#endif // ASSERT

// Create an Initialized Assertion Predicate from the template_assertion_predicate
IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
Expand Down Expand Up @@ -1978,55 +1967,32 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_
}
}

void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block,
LoopNode* outer_loop_head,
const int dd_outer_loop_head, Node* init,
Node* stride, IdealLoopTree* outer_loop,
const uint idx_before_clone,
const Node_List &old_new) {
if (!predicate_block->has_parse_predicate()) {
return;
}
Node* input_proj = outer_loop_head->in(LoopNode::EntryControl);
const Node* parse_predicate_uncommon_trap = predicate_block->parse_predicate()->uncommon_trap();
Node* next_regular_predicate_proj = predicate_block->skip_parse_predicate();
while (next_regular_predicate_proj->is_IfProj()) {
IfNode* iff = next_regular_predicate_proj->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - next_regular_predicate_proj->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != parse_predicate_uncommon_trap) {
// Does not belong to this Predicate Block anymore.
break;
}
Node* bol = iff->in(1);
assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate");
if (bol->is_OpaqueTemplateAssertionPredicate()) {
// Initialize from Template Assertion Predicate.
input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj);

// Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized
// Assertion Predicates above the peeled loop.
for (DUIterator i = next_regular_predicate_proj->outs(); next_regular_predicate_proj->has_out(i); i++) {
Node* dependent = next_regular_predicate_proj->out(i);
Node* new_node = old_new[dependent->_idx];

if (!dependent->is_CFG() &&
dependent->_idx < idx_before_clone && // old node
new_node != nullptr && // cloned
new_node->_idx >= idx_before_clone) { // for peeling
// The old nodes from the peeled loop still point to the predicate above the peeled loop.
// We need to rewire the dependencies to the newly Initialized Assertion Predicates.
_igvn.replace_input_of(dependent, 0, input_proj);
--i; // correct for just deleted predicate->out(i)
}
}
}
next_regular_predicate_proj = iff->in(0);
}

_igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj);
set_idom(outer_loop_head, input_proj, dd_outer_loop_head);
void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
CountedLoopNode* remaining_loop_head,
const uint first_node_index_in_cloned_loop_body,
const Node_List& old_new) {
const NodeInOriginalLoopBody node_in_original_loop_body(first_node_index_in_cloned_loop_body, old_new);
create_assertion_predicates_at_loop(peeled_loop_head, remaining_loop_head, node_in_original_loop_body);
}

void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head,
CountedLoopNode* target_loop_head,
const NodeInLoopBody& _node_in_loop_body) {
Node* init = target_loop_head->init_trip();
Node* stride = target_loop_head->stride();
LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined();
Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl);
AssertionPredicatesForLoop assertion_predicates_for_loop(init, stride, target_loop_entry, this,
_node_in_loop_body);
Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
PredicateIterator predicate_iterator(source_loop_entry);
predicate_iterator.for_each(assertion_predicates_for_loop);
if (assertion_predicates_for_loop.has_created_predicates()) {
IfTrueNode* last_created_node = assertion_predicates_for_loop.last_created_success_proj();
_igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_node);
set_idom(target_outer_loop_head, last_created_node, dom_depth(target_outer_loop_head));
}
}
//------------------------------do_unroll--------------------------------------
// Unroll the loop body one step - make each trip do 2 iterations.
void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip) {
Expand Down
12 changes: 8 additions & 4 deletions src/hotspot/share/opto/loopnode.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -952,18 +952,22 @@ class PhaseIdealLoop : public PhaseTransform {
Node* zero_trip_guard_proj_post, const Node_List& old_new);
Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* new_control);
public:
IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
Node* new_stride, Node* control);
static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
private:
DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);)
DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);)
static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head,
Node* stride);
void initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, LoopNode* outer_loop_head,
int dd_outer_loop_head, Node* init, Node* stride,
IdealLoopTree* outer_loop, uint idx_before_clone,
void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
CountedLoopNode* remaining_loop_head,
uint first_node_index_in_cloned_loop_body,
const Node_List& old_new);
void create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head,
const NodeInLoopBody& _node_in_loop_body);
void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit,
Node* bol);
void log_loop_tree();
Expand Down
36 changes: 36 additions & 0 deletions src/hotspot/share/opto/predicates.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,21 @@ bool RuntimePredicate::is_predicate(Node* node, Deoptimization::DeoptReason deop
return RegularPredicateWithUCT::is_predicate(node, deopt_reason);
}

// Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized
// Assertion Predicates above the peeled loop.
void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* target_predicate,
const NodeInLoopBody& _data_in_loop_body,
PhaseIdealLoop* phase) const {
for (DUIterator i = _success_proj->outs(); _success_proj->has_out(i); i++) {
Node* output = _success_proj->out(i);
if (!output->is_CFG() && _data_in_loop_body.check(output)) {
phase->igvn().replace_input_of(output, 0, target_predicate);
--i; // correct for just deleted output
}
}
}


// Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them.
bool TemplateAssertionPredicate::is_predicate(Node* node) {
if (!may_be_assertion_predicate_if(node)) {
Expand Down Expand Up @@ -712,3 +727,24 @@ void Predicates::dump_for_loop(LoopNode* loop_node) {
dump_at(loop_node->skip_strip_mined()->in(LoopNode::EntryControl));
}
#endif // NOT PRODUCT

// Keep track of the current Predicate Block by setting '_current_parse_predicate'.
void AssertionPredicatesForLoop::visit(const ParsePredicate& parse_predicate) {
Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason();
if (deopt_reason == Deoptimization::Reason_predicate ||
deopt_reason == Deoptimization::Reason_profile_predicate) {
_current_parse_predicate = parse_predicate.tail();
}
}

void AssertionPredicatesForLoop::visit(const TemplateAssertionPredicate& template_assertion_predicate) {
if (_current_parse_predicate == nullptr) {
// Only process if we are in the correct Predicate Block.
return;
}
IfNode* template_head = template_assertion_predicate.head();
IfTrueNode* initialized_predicate = _phase->create_initialized_assertion_predicate(template_head, _init, _stride,
_new_control);
template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase);
_new_control = initialized_predicate;
}
75 changes: 75 additions & 0 deletions src/hotspot/share/opto/predicates.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,12 @@ class PredicateVisitor : StackObj {
}
};

// Interface to check whether a node is in a loop body or not.
class NodeInLoopBody : public StackObj {
public:
virtual bool check(Node* node) const = 0;
};

// Class to represent Assertion Predicates with a HaltNode instead of an UCT (i.e. either an Initialized Assertion
// Predicate or a Template Assertion Predicate created after the initial one at Loop Predication).
class AssertionPredicatesWithHalt : public StackObj {
Expand Down Expand Up @@ -393,6 +399,9 @@ class TemplateAssertionPredicate : public Predicate {
return _success_proj;
}

void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& _data_in_loop_body,
PhaseIdealLoop* phase) const;

static bool is_predicate(Node* node);
};

Expand Down Expand Up @@ -910,4 +919,70 @@ class Predicates : public StackObj {
#endif // NOT PRODUCT
};

// This class checks whether a node is in the original loop body and not the cloned one.
class NodeInOriginalLoopBody : public NodeInLoopBody {
const uint _first_node_index_in_cloned_loop_body;
const Node_List& _old_new;

public:
NodeInOriginalLoopBody(const uint first_node_index_in_cloned_loop_body, const Node_List& old_new)
: _first_node_index_in_cloned_loop_body(first_node_index_in_cloned_loop_body),
_old_new(old_new) {}
NONCOPYABLE(NodeInOriginalLoopBody);

// Check if 'node' is not a cloned node (i.e. "< _first_node_index_in_cloned_loop_body") and if we've created a
// clone from 'node' (i.e. _old_new entry is non-null). Then we know that 'node' belongs to the original loop body.
bool check(Node* node) const override {
if (node->_idx < _first_node_index_in_cloned_loop_body) {
Node* cloned_node = _old_new[node->_idx];
return cloned_node != nullptr && cloned_node->_idx >= _first_node_index_in_cloned_loop_body;
} else {
return false;
}
}
};

// Visitor to create Initialized Assertion Predicates at a target from Template Assertion Predicates from a source loop.
// This visitor can be used in combination with a PredicateIterator.
class AssertionPredicatesForLoop : public PredicateVisitor {
Node* const _init;
Node* const _stride;
Node* const _old_target_loop_entry;
Node* _new_control;
PhaseIdealLoop* const _phase;
ParsePredicateSuccessProj* _current_parse_predicate;
const NodeInLoopBody& _node_in_loop_body;

public:
AssertionPredicatesForLoop(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase,
const NodeInLoopBody& node_in_loop_body)
: _init(init),
_stride(stride),
_old_target_loop_entry(new_control),
_new_control(new_control),
_phase(phase),
_current_parse_predicate(nullptr),
_node_in_loop_body(node_in_loop_body) {}
NONCOPYABLE(AssertionPredicatesForLoop);

using PredicateVisitor::visit;

void visit(const ParsePredicate& parse_predicate) override;
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;

// Did we create any new Initialized Assertion Predicates?
bool has_created_predicates() const {
return _new_control != _old_target_loop_entry;
}

// Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was
// no new node created (i.e. no Template Assertion Predicates found).
IfTrueNode* last_created_success_proj() const {
assert(has_created_predicates(), "should only be queried if new nodes have been created");
assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet");
assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj");
return _new_control->as_IfTrue();
}
};

#endif // SHARE_OPTO_PREDICATES_HPP

0 comments on commit aeca241

Please sign in to comment.