From 978a33ae82ae76d95248dcc3384ecb61c344a55e Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Thu, 14 Nov 2024 14:31:03 +0100 Subject: [PATCH] 8344171: Clone and initialize Assertion Predicates in order instead of in reverse-order --- src/hotspot/share/opto/loopTransform.cpp | 13 +------ src/hotspot/share/opto/loopnode.hpp | 6 +++ src/hotspot/share/opto/predicates.cpp | 48 +++++++++++++++++++----- src/hotspot/share/opto/predicates.hpp | 32 +++------------- 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index f644e26bbe77f..88f07ac1f990a 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1753,20 +1753,11 @@ void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source CountedLoopNode* target_loop_head, const NodeInLoopBody& _node_in_loop_body, const bool clone_template) { - Node* init = target_loop_head->init_trip(); - Node* stride = target_loop_head->stride(); - LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined(); - Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl); - CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this, - _node_in_loop_body, clone_template); + CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(target_loop_head, this, _node_in_loop_body, + clone_template); Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); PredicateIterator predicate_iterator(source_loop_entry); predicate_iterator.for_each(create_assertion_predicates_visitor); - if (create_assertion_predicates_visitor.has_created_predicates()) { - IfTrueNode* last_created_predicate_success_proj = create_assertion_predicates_visitor.last_created_success_proj(); - _igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_predicate_success_proj); - set_idom(target_outer_loop_head, last_created_predicate_success_proj, dom_depth(target_outer_loop_head)); - } } //------------------------------do_unroll-------------------------------------- // Unroll the loop body one step - make each trip do 2 iterations. diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 698e48aadb4ef..9a9621e6de074 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1363,6 +1363,12 @@ class PhaseIdealLoop : public PhaseTransform { public: void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true); + // Replace the control input of 'node' with 'new_control' and set the dom depth to the one of 'new_control'. + void replace_control_same_depth(Node* node, Node* new_control) { + _igvn.replace_input_of(node, 0, new_control); + set_idom(node, new_control, dom_depth(new_control)); + } + void replace_loop_entry(LoopNode* loop_head, Node* new_entry) { _igvn.replace_input_of(loop_head, LoopNode::EntryControl, new_entry); set_idom(loop_head, new_entry, dom_depth(new_entry)); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index e11664bd35911..99f6931abcf0a 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -879,6 +879,19 @@ void Predicates::dump_for_loop(LoopNode* loop_node) { } #endif // NOT PRODUCT +CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, + PhaseIdealLoop* phase, + const NodeInLoopBody& node_in_loop_body, + const bool clone_template) + : _init(target_loop_head->init_trip()), + _stride(target_loop_head->stride()), + _old_target_loop_entry(target_loop_head->skip_strip_mined()->in(LoopNode::EntryControl)), + _current_predicate_chain_head(target_loop_head->skip_strip_mined()), // Initially no predicates, yet. + _phase(phase), + _has_hoisted_check_parse_predicates(false), + _node_in_loop_body(node_in_loop_body), + _clone_template(clone_template) {} + // Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found. // The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block. void CreateAssertionPredicatesVisitor::visit(const ParsePredicate& parse_predicate) { @@ -895,21 +908,26 @@ void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& t return; } if (_clone_template) { - _new_control = clone_template_and_replace_init_input(template_assertion_predicate); + IfTrueNode* cloned_template_success_proj = clone_template_and_replace_init_input(template_assertion_predicate); + initialize_from_template(template_assertion_predicate, cloned_template_success_proj); + _current_predicate_chain_head = cloned_template_success_proj->in(0); + } else { + IfTrueNode* initialized_success_proj = initialize_from_template(template_assertion_predicate, _old_target_loop_entry); + _current_predicate_chain_head = initialized_success_proj->in(0); } - _new_control = initialize_from_template(template_assertion_predicate); } // Create an Initialized Assertion Predicate from the provided Template Assertion Predicate. IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template( - const TemplateAssertionPredicate& template_assertion_predicate) const { + const TemplateAssertionPredicate& template_assertion_predicate, Node* new_control) const { DEBUG_ONLY(template_assertion_predicate.verify();) IfNode* template_head = template_assertion_predicate.head(); InitializedAssertionPredicateCreator initialized_assertion_predicate(_phase); - IfTrueNode* initialized_predicate = initialized_assertion_predicate.create_from_template(template_head,_new_control, + IfTrueNode* initialized_predicate = initialized_assertion_predicate.create_from_template(template_head, new_control, _init, _stride); DEBUG_ONLY(InitializedAssertionPredicate::verify(initialized_predicate);) template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase); + rewire_to_old_predicate_chain_head(initialized_predicate); return initialized_predicate; } @@ -917,8 +935,21 @@ IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template( IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_input( const TemplateAssertionPredicate& template_assertion_predicate) { OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _init); - _phase->register_new_node(opaque_init, _new_control); - return template_assertion_predicate.clone_and_replace_init(_new_control, opaque_init, _phase); + _phase->register_new_node(opaque_init, _old_target_loop_entry); + return template_assertion_predicate.clone_and_replace_init(_old_target_loop_entry, opaque_init, _phase); +} + +// Rewire the newly created predicates to the old predicate chain head (i.e. '_current_predicate_chain_head') by +// rewiring its control input from '_old_target_loop_entry' to 'initialized_assertion_predicate_success_proj'. +void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head( + Node* initialized_assertion_predicate_success_proj) const { + if (_current_predicate_chain_head->is_Loop()) { + assert(_current_predicate_chain_head->in(LoopNode::EntryControl) == _old_target_loop_entry, "must be old loop entry"); + _phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), initialized_assertion_predicate_success_proj); + } else { + assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry, "must be old loop entry"); + _phase->replace_control_same_depth(_current_predicate_chain_head, initialized_assertion_predicate_success_proj); + } } // Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode. @@ -951,9 +982,8 @@ IfTrueNode* UpdateStrideForAssertionPredicates::initialize_from_updated_template void UpdateStrideForAssertionPredicates::connect_initialized_assertion_predicate( Node* new_control_out, IfTrueNode* initialized_success_proj) const { if (new_control_out->is_Loop()) { - _phase->igvn().replace_input_of(new_control_out, LoopNode::EntryControl, initialized_success_proj); + _phase->replace_loop_entry(new_control_out->as_Loop(), initialized_success_proj); } else { - _phase->igvn().replace_input_of(new_control_out, 0, initialized_success_proj); + _phase->replace_control_same_depth(new_control_out, initialized_success_proj); } - _phase->set_idom(new_control_out, initialized_success_proj, _phase->dom_depth(new_control_out)); } diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 407a931a91ca7..4cd909376aa0d 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -984,46 +984,26 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { Node* const _init; Node* const _stride; Node* const _old_target_loop_entry; - Node* _new_control; + Node* _current_predicate_chain_head; PhaseIdealLoop* const _phase; bool _has_hoisted_check_parse_predicates; const NodeInLoopBody& _node_in_loop_body; const bool _clone_template; IfTrueNode* clone_template_and_replace_init_input(const TemplateAssertionPredicate& template_assertion_predicate); - IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate) const; + IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate, + Node* new_control) const; + void rewire_to_old_predicate_chain_head(Node* initialized_assertion_predicate_success_proj) const; public: - CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase, - const NodeInLoopBody& node_in_loop_body, const bool clone_template) - : _init(init), - _stride(stride), - _old_target_loop_entry(new_control), - _new_control(new_control), - _phase(phase), - _has_hoisted_check_parse_predicates(false), - _node_in_loop_body(node_in_loop_body), - _clone_template(clone_template) {} + CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase, + const NodeInLoopBody& node_in_loop_body, bool clone_template); NONCOPYABLE(CreateAssertionPredicatesVisitor); using PredicateVisitor::visit; void visit(const ParsePredicate& parse_predicate) override; void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; - - // Did we create any new Initialized Assertion Predicates? - bool has_created_predicates() const { - return _new_control != _old_target_loop_entry; - } - - // Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was - // no new node created (i.e. no Template Assertion Predicates found). - IfTrueNode* last_created_success_proj() const { - assert(has_created_predicates(), "should only be queried if new nodes have been created"); - assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet"); - assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj"); - return _new_control->as_IfTrue(); - } }; // This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the