From 63c19d3db586920108808866c7a094a5ae41bc22 Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Wed, 30 Oct 2024 06:11:06 +0000 Subject: [PATCH] 8341977: Replace predicate walking and cloning code for Loop Peeling with a predicate visitor Reviewed-by: kvn, epeter --- src/hotspot/share/opto/loopTransform.cpp | 90 ++++++++---------------- src/hotspot/share/opto/loopnode.hpp | 12 ++-- src/hotspot/share/opto/predicates.cpp | 37 ++++++++++ src/hotspot/share/opto/predicates.hpp | 74 +++++++++++++++++++ 4 files changed, 147 insertions(+), 66 deletions(-) diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 322183ec3a092..f5a7ffcf92ada 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -762,7 +762,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 1: Clone the loop body. The clone becomes the peeled iteration. // The pre-loop illegally has 2 control users (old & new loops). - const uint idx_before_clone = Compile::current()->unique(); + const uint first_node_index_in_cloned_loop_body = Compile::current()->unique(); LoopNode* outer_loop_head = head->skip_strip_mined(); clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined); @@ -815,19 +815,8 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 5: Assertion Predicates initialization if (counted_loop && UseLoopPredicate) { - CountedLoopNode *cl_head = head->as_CountedLoop(); - Node* init = cl_head->init_trip(); - Node* stride = cl_head->stride(); - IdealLoopTree* outer_loop = get_loop(outer_loop_head); - const Predicates predicates(new_head->in(LoopNode::EntryControl)); - initialize_assertion_predicates_for_peeled_loop(predicates.loop_predicate_block(), - outer_loop_head, dd_outer_loop_head, - init, stride, outer_loop, - idx_before_clone, old_new); - initialize_assertion_predicates_for_peeled_loop(predicates.profiled_loop_predicate_block(), - outer_loop_head, dd_outer_loop_head, - init, stride, outer_loop, - idx_before_clone, old_new); + initialize_assertion_predicates_for_peeled_loop(new_head->as_CountedLoop(), head->as_CountedLoop(), + first_node_index_in_cloned_loop_body, old_new); } // Now force out all loop-invariant dominating tests. The optimizer @@ -1434,7 +1423,6 @@ bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { assert(stride == 0 || init != 0, "init should be there every time stride is"); return init != 0; } -#endif // ASSERT void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) { init = 0; @@ -1460,6 +1448,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) } } } +#endif // ASSERT // Create an Initialized Assertion Predicate from the template_assertion_predicate IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, @@ -1978,55 +1967,32 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_ } } -void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, - LoopNode* outer_loop_head, - const int dd_outer_loop_head, Node* init, - Node* stride, IdealLoopTree* outer_loop, - const uint idx_before_clone, - const Node_List &old_new) { - if (!predicate_block->has_parse_predicate()) { - return; - } - Node* input_proj = outer_loop_head->in(LoopNode::EntryControl); - const Node* parse_predicate_uncommon_trap = predicate_block->parse_predicate()->uncommon_trap(); - Node* next_regular_predicate_proj = predicate_block->skip_parse_predicate(); - while (next_regular_predicate_proj->is_IfProj()) { - IfNode* iff = next_regular_predicate_proj->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - next_regular_predicate_proj->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != parse_predicate_uncommon_trap) { - // Does not belong to this Predicate Block anymore. - break; - } - Node* bol = iff->in(1); - assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_OpaqueTemplateAssertionPredicate()) { - // Initialize from Template Assertion Predicate. - input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj); - - // Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized - // Assertion Predicates above the peeled loop. - for (DUIterator i = next_regular_predicate_proj->outs(); next_regular_predicate_proj->has_out(i); i++) { - Node* dependent = next_regular_predicate_proj->out(i); - Node* new_node = old_new[dependent->_idx]; - - if (!dependent->is_CFG() && - dependent->_idx < idx_before_clone && // old node - new_node != nullptr && // cloned - new_node->_idx >= idx_before_clone) { // for peeling - // The old nodes from the peeled loop still point to the predicate above the peeled loop. - // We need to rewire the dependencies to the newly Initialized Assertion Predicates. - _igvn.replace_input_of(dependent, 0, input_proj); - --i; // correct for just deleted predicate->out(i) - } - } - } - next_regular_predicate_proj = iff->in(0); - } - - _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj); - set_idom(outer_loop_head, input_proj, dd_outer_loop_head); +void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, + CountedLoopNode* remaining_loop_head, + const uint first_node_index_in_cloned_loop_body, + const Node_List& old_new) { + const NodeInOriginalLoopBody node_in_original_loop_body(first_node_index_in_cloned_loop_body, old_new); + create_assertion_predicates_at_loop(peeled_loop_head, remaining_loop_head, node_in_original_loop_body); } +void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, + CountedLoopNode* target_loop_head, + const NodeInLoopBody& _node_in_loop_body) { + Node* init = target_loop_head->init_trip(); + Node* stride = target_loop_head->stride(); + LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined(); + Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl); + CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this, + _node_in_loop_body); + Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); + PredicateIterator predicate_iterator(source_loop_entry); + predicate_iterator.for_each(create_assertion_predicates_visitor); + if (create_assertion_predicates_visitor.has_created_predicates()) { + IfTrueNode* last_created_predicate_success_proj = create_assertion_predicates_visitor.last_created_success_proj(); + _igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_predicate_success_proj); + set_idom(target_outer_loop_head, last_created_predicate_success_proj, dom_depth(target_outer_loop_head)); + } +} //------------------------------do_unroll-------------------------------------- // Unroll the loop body one step - make each trip do 2 iterations. void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip) { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index a2094045bfa94..fc0f1b4d53c5e 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -952,18 +952,22 @@ class PhaseIdealLoop : public PhaseTransform { Node* zero_trip_guard_proj_post, const Node_List& old_new); Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, Node* new_control); + public: IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, Node* control); - static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); + private: + DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);) DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* stride); - void initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, LoopNode* outer_loop_head, - int dd_outer_loop_head, Node* init, Node* stride, - IdealLoopTree* outer_loop, uint idx_before_clone, + void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, + CountedLoopNode* remaining_loop_head, + uint first_node_index_in_cloned_loop_body, const Node_List& old_new); + void create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head, + const NodeInLoopBody& _node_in_loop_body); void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, Node* bol); void log_loop_tree(); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index fd1aa1a9cd1ca..54a17376f18e4 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -138,6 +138,21 @@ bool RuntimePredicate::is_predicate(Node* node, Deoptimization::DeoptReason deop return RegularPredicateWithUCT::is_predicate(node, deopt_reason); } +// Rewire any non-CFG nodes dependent on this Template Assertion Predicate (i.e. with a control input to this +// Template Assertion Predicate) to the 'target_predicate' based on the 'data_in_loop_body' check. +void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* target_predicate, + const NodeInLoopBody& data_in_loop_body, + PhaseIdealLoop* phase) const { + for (DUIterator i = _success_proj->outs(); _success_proj->has_out(i); i++) { + Node* output = _success_proj->out(i); + if (!output->is_CFG() && data_in_loop_body.check(output)) { + phase->igvn().replace_input_of(output, 0, target_predicate); + --i; // account for the just deleted output + } + } +} + + // Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them. bool TemplateAssertionPredicate::is_predicate(Node* node) { if (!may_be_assertion_predicate_if(node)) { @@ -712,3 +727,25 @@ void Predicates::dump_for_loop(LoopNode* loop_node) { dump_at(loop_node->skip_strip_mined()->in(LoopNode::EntryControl)); } #endif // NOT PRODUCT + +// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found. +// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block. +void CreateAssertionPredicatesVisitor::visit(const ParsePredicate& parse_predicate) { + Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason(); + if (deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate) { + _has_hoisted_check_parse_predicates = true; + } +} + +void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) { + if (!_has_hoisted_check_parse_predicates) { + // Only process if we are in the correct Predicate Block. + return; + } + IfNode* template_head = template_assertion_predicate.head(); + IfTrueNode* initialized_predicate = _phase->create_initialized_assertion_predicate(template_head, _init, _stride, + _new_control); + template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase); + _new_control = initialized_predicate; +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index ced401a5f7776..5eb0cfc9b3578 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -248,6 +248,12 @@ class PredicateVisitor : StackObj { } }; +// Interface to check whether a node is in a loop body or not. +class NodeInLoopBody : public StackObj { + public: + virtual bool check(Node* node) const = 0; +}; + // Class to represent Assertion Predicates with a HaltNode instead of an UCT (i.e. either an Initialized Assertion // Predicate or a Template Assertion Predicate created after the initial one at Loop Predication). class AssertionPredicatesWithHalt : public StackObj { @@ -393,6 +399,8 @@ class TemplateAssertionPredicate : public Predicate { return _success_proj; } + void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, + PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); }; @@ -910,4 +918,70 @@ class Predicates : public StackObj { #endif // NOT PRODUCT }; +// This class checks whether a node is in the original loop body and not the cloned one. +class NodeInOriginalLoopBody : public NodeInLoopBody { + const uint _first_node_index_in_cloned_loop_body; + const Node_List& _old_new; + + public: + NodeInOriginalLoopBody(const uint first_node_index_in_cloned_loop_body, const Node_List& old_new) + : _first_node_index_in_cloned_loop_body(first_node_index_in_cloned_loop_body), + _old_new(old_new) {} + NONCOPYABLE(NodeInOriginalLoopBody); + + // Check if 'node' is not a cloned node (i.e. "< _first_node_index_in_cloned_loop_body") and if we've created a + // clone from 'node' (i.e. _old_new entry is non-null). Then we know that 'node' belongs to the original loop body. + bool check(Node* node) const override { + if (node->_idx < _first_node_index_in_cloned_loop_body) { + Node* cloned_node = _old_new[node->_idx]; + return cloned_node != nullptr && cloned_node->_idx >= _first_node_index_in_cloned_loop_body; + } else { + return false; + } + } +}; + +// Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source +// loop. This visitor can be used in combination with a PredicateIterator. +class CreateAssertionPredicatesVisitor : public PredicateVisitor { + Node* const _init; + Node* const _stride; + Node* const _old_target_loop_entry; + Node* _new_control; + PhaseIdealLoop* const _phase; + bool _has_hoisted_check_parse_predicates; + const NodeInLoopBody& _node_in_loop_body; + + public: + CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase, + const NodeInLoopBody& node_in_loop_body) + : _init(init), + _stride(stride), + _old_target_loop_entry(new_control), + _new_control(new_control), + _phase(phase), + _has_hoisted_check_parse_predicates(false), + _node_in_loop_body(node_in_loop_body) {} + NONCOPYABLE(CreateAssertionPredicatesVisitor); + + using PredicateVisitor::visit; + + void visit(const ParsePredicate& parse_predicate) override; + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; + + // Did we create any new Initialized Assertion Predicates? + bool has_created_predicates() const { + return _new_control != _old_target_loop_entry; + } + + // Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was + // no new node created (i.e. no Template Assertion Predicates found). + IfTrueNode* last_created_success_proj() const { + assert(has_created_predicates(), "should only be queried if new nodes have been created"); + assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet"); + assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj"); + return _new_control->as_IfTrue(); + } +}; + #endif // SHARE_OPTO_PREDICATES_HPP