From 2eef967423a9488d02b6ceef52f66bb1518aa311 Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Wed, 23 Oct 2024 15:15:53 +0200 Subject: [PATCH] 8342943: Replace predicate walking and cloning code for main/post loops with a predicate visitor --- src/hotspot/share/opto/loopPredicate.cpp | 1 - src/hotspot/share/opto/loopTransform.cpp | 194 ++++------------------- src/hotspot/share/opto/loopnode.hpp | 25 ++- src/hotspot/share/opto/predicates.cpp | 36 ++++- src/hotspot/share/opto/predicates.hpp | 30 +++- 5 files changed, 105 insertions(+), 181 deletions(-) diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index d1de9c981012c..9a639b1f9a1f2 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -24,7 +24,6 @@ #include "precompiled.hpp" #include "memory/allocation.hpp" -#include "opto/loopnode.hpp" #include "opto/addnode.hpp" #include "opto/callnode.hpp" #include "opto/castnode.hpp" diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index f5a7ffcf92ada..c88ffdbad820f 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -762,7 +762,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 1: Clone the loop body. The clone becomes the peeled iteration. // The pre-loop illegally has 2 control users (old & new loops). - const uint first_node_index_in_cloned_loop_body = Compile::current()->unique(); + const uint first_node_index_in_post_loop_body = Compile::current()->unique(); LoopNode* outer_loop_head = head->skip_strip_mined(); clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined); @@ -816,7 +816,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 5: Assertion Predicates initialization if (counted_loop && UseLoopPredicate) { initialize_assertion_predicates_for_peeled_loop(new_head->as_CountedLoop(), head->as_CountedLoop(), - first_node_index_in_cloned_loop_body, old_new); + first_node_index_in_post_loop_body, old_new); } // Now force out all loop-invariant dominating tests. The optimizer @@ -1313,83 +1313,6 @@ void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop) } #endif -// Make two copies of each Template Assertion Predicate before the pre-loop and add them to the main-loop. One remains -// a template while the other one is initialized with the initial value of the loop induction variable. The Initialized -// Assertion Predicates ensures that the main-loop is removed if some type ranges of Cast or Convert nodes become -// impossible and are replaced by top (i.e. a sign that the main-loop is dead). -void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const PredicateBlock* predicate_block, Node* init, - Node* stride, IdealLoopTree* outer_loop, - LoopNode* outer_main_head, const uint dd_main_head, - const uint idx_before_pre_post, - const uint idx_after_post_before_pre, - Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, - const Node_List &old_new) { - if (predicate_block->has_parse_predicate()) { -#ifdef ASSERT - ensure_zero_trip_guard_proj(zero_trip_guard_proj_main, true); - ensure_zero_trip_guard_proj(zero_trip_guard_proj_post, false); -#endif - Node* predicate_proj = predicate_block->parse_predicate_success_proj(); - IfNode* iff = predicate_proj->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->as_Proj()->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - predicate_proj = iff->in(0); - Node* current_proj = outer_main_head->in(LoopNode::EntryControl); - Node* prev_proj = current_proj; - Node* opaque_init = new OpaqueLoopInitNode(C, init); - register_new_node(opaque_init, outer_main_head->in(LoopNode::EntryControl)); - Node* opaque_stride = new OpaqueLoopStrideNode(C, stride); - register_new_node(opaque_stride, outer_main_head->in(LoopNode::EntryControl)); - - while (predicate_proj != nullptr && predicate_proj->is_Proj() && predicate_proj->in(0)->is_If()) { - iff = predicate_proj->in(0)->as_If(); - uncommon_proj = iff->proj_out(1 - predicate_proj->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != rgn) - break; - Node* bol = iff->in(1); - assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_OpaqueTemplateAssertionPredicate()) { - // Clone the Assertion Predicate twice and initialize one with the initial - // value of the loop induction variable. Leave the other predicate - // to be initialized when increasing the stride during loop unrolling. - prev_proj = clone_template_assertion_predicate(iff, opaque_init, predicate_proj, uncommon_proj, - current_proj, outer_loop, prev_proj); - prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj); - - // Rewire any control inputs from the cloned Assertion Predicates down to the main and post loop for data nodes - // that are part of the main loop (and were cloned to the pre and post loop). - for (DUIterator i = predicate_proj->outs(); predicate_proj->has_out(i); i++) { - Node* loop_node = predicate_proj->out(i); - Node* pre_loop_node = old_new[loop_node->_idx]; - // Change the control if 'loop_node' is part of the main loop. If there is an old->new mapping and the index of - // 'pre_loop_node' is greater than idx_before_pre_post, then we know that 'loop_node' was cloned and is part of - // the main loop (and 'pre_loop_node' is part of the pre loop). - if (!loop_node->is_CFG() && (pre_loop_node != nullptr && pre_loop_node->_idx > idx_after_post_before_pre)) { - // 'loop_node' is a data node and part of the main loop. Rewire the control to the projection of the zero-trip guard if node - // of the main loop that is immediately preceding the cloned predicates. - _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_main); - --i; - } else if (loop_node->_idx > idx_before_pre_post && loop_node->_idx < idx_after_post_before_pre) { - // 'loop_node' is a data node and part of the post loop. Rewire the control to the projection of the zero-trip guard if node - // of the post loop that is immediately preceding the post loop header node (there are no cloned predicates for the post loop). - assert(pre_loop_node == nullptr, "a node belonging to the post loop should not have an old_new mapping at this stage"); - _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_post); - --i; - } - } - - // Remove the Assertion Predicate from the pre-loop - _igvn.replace_input_of(iff, 1, _igvn.intcon(1)); - } - predicate_proj = predicate_proj->in(0)->in(0); - } - _igvn.replace_input_of(outer_main_head, LoopNode::EntryControl, prev_proj); - set_idom(outer_main_head, prev_proj, dd_main_head); - } -} - #ifdef ASSERT bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { uint init; @@ -1464,47 +1387,6 @@ IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* templ return success_proj; } -// Clone the Template Assertion Predicate and set a new OpaqueLoopInitNode to create a new Template Assertion Predicate. -// This is done when creating a new Template Assertion Predicate for the main loop which requires a new init node. -// We keep the OpaqueTemplateAssertionPredicate node since it's still a template. Since the templates are eventually -// removed after loop opts, these are never executed. We therefore insert a Halt node instead of an uncommon trap. -Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, - Node* control, IdealLoopTree* outer_loop, Node* new_control) { - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate"); - TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_OpaqueTemplateAssertionPredicate()); - assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); - OpaqueTemplateAssertionPredicateNode* new_opaque_node = - template_assertion_expression.clone_and_replace_init(new_init, control, this); - AssertionPredicateIfCreator assertion_predicate_if_creator(this); - IfTrueNode* success_proj = - assertion_predicate_if_creator.create_for_template(new_control, iff->Opcode(), new_opaque_node - NOT_PRODUCT(COMMA iff->assertion_predicate_type())); - assert(assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), - "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); - return success_proj; -} - -void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, - IdealLoopTree* outer_loop, LoopNode* outer_main_head, - const uint dd_main_head, const uint idx_before_pre_post, - const uint idx_after_post_before_pre, - Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, - const Node_List &old_new) { - if (UseLoopPredicate) { - Node* entry = pre_head->in(LoopNode::EntryControl); - const Predicates predicates(entry); - copy_assertion_predicates_to_main_loop_helper(predicates.loop_predicate_block(), init, stride, outer_loop, - outer_main_head, dd_main_head, idx_before_pre_post, - idx_after_post_before_pre, zero_trip_guard_proj_main, - zero_trip_guard_proj_post, old_new); - copy_assertion_predicates_to_main_loop_helper(predicates.profiled_loop_predicate_block(), init, stride, - outer_loop, outer_main_head, dd_main_head, idx_before_pre_post, - idx_after_post_before_pre, zero_trip_guard_proj_main, - zero_trip_guard_proj_post, old_new); - } -} - //------------------------------insert_pre_post_loops-------------------------- // Insert pre and post loops. If peel_only is set, the pre-loop can not have // more iterations added. It acts as a 'peel' only, no lower-bound RCE, no @@ -1553,11 +1435,9 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n } // Add the post loop - const uint idx_before_pre_post = Compile::current()->unique(); CountedLoopNode *post_head = nullptr; Node* post_incr = incr; Node* main_exit = insert_post_loop(loop, old_new, main_head, main_end, post_incr, limit, post_head); - const uint idx_after_post_before_pre = Compile::current()->unique(); //------------------------------ // Step B: Create Pre-Loop. @@ -1572,6 +1452,8 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n outer_loop = loop->_parent; assert(outer_loop->_head == outer_main_head, "broken loop tree"); } + + const uint first_node_index_in_pre_loop_body = Compile::current()->unique(); uint dd_main_head = dom_depth(outer_main_head); clone_loop(loop, old_new, dd_main_head, ControlAroundStripMined); CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop(); @@ -1650,10 +1532,10 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // dependencies. assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop"); - copy_assertion_predicates_to_main_loop(pre_head, pre_incr, stride, outer_loop, outer_main_head, dd_main_head, - idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), - old_new); - copy_assertion_predicates_to_post_loop(outer_main_head, post_head, stride); + DEBUG_ONLY(ensure_zero_trip_guard_proj(outer_main_head->in(LoopNode::EntryControl), true);) + if (UseLoopPredicate) { + initialize_assertion_predicates_for_main_loop(main_head, pre_head, first_node_index_in_pre_loop_body, old_new); + } // Step B4: Shorten the pre-loop to run only 1 iteration (for now). // RCE and alignment may change this later. @@ -1778,7 +1660,6 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old // In this case we throw away the result as we are not using it to connect anything else. CountedLoopNode *post_head = nullptr; insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head); - copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, main_head->stride()); // It's difficult to be precise about the trip-counts // for post loops. They are usually very short, @@ -1813,6 +1694,7 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, // Step A1: Clone the loop body of main. The clone becomes the post-loop. // The main loop pre-header illegally has 2 control users (old & new loops). + const uint first_node_index_in_cloned_loop_body = C->unique(); clone_loop(loop, old_new, dd_main_exit, ControlAroundStripMined); assert(old_new[main_end->_idx]->Opcode() == Op_CountedLoopEnd, ""); post_head = old_new[main_head->_idx]->as_CountedLoop(); @@ -1881,6 +1763,10 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, } } + DEBUG_ONLY(ensure_zero_trip_guard_proj(post_head->in(LoopNode::EntryControl), false);) + if (UseLoopPredicate) { + initialize_assertion_predicates_for_post_loop(main_head, post_head, first_node_index_in_cloned_loop_body); + } return new_main_exit; } @@ -1937,53 +1823,39 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo } } -// Go over the Assertion Predicates of the main loop and make a copy for the post loop with its initial iv value and -// stride as inputs. -void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, - Node* stride) { - Node* opaq = post_loop_head->is_canonical_loop_entry(); - Node* init = opaq->in(1); - Node* post_loop_entry = post_loop_head->in(LoopNode::EntryControl); - Node* main_loop_entry = main_loop_head->in(LoopNode::EntryControl); - IdealLoopTree* post_loop = get_loop(post_loop_head); - - Node* ctrl = main_loop_entry; - Node* prev_proj = post_loop_entry; - while (ctrl != nullptr && ctrl->is_Proj() && ctrl->in(0)->is_If()) { - IfNode* iff = ctrl->in(0)->as_If(); - ProjNode* proj = iff->proj_out(1 - ctrl->as_Proj()->_con); - if (!proj->unique_ctrl_out()->is_Halt()) { - break; - } - if (iff->in(1)->is_OpaqueTemplateAssertionPredicate()) { - // Initialize from Template Assertion Predicate. - prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj); - } - ctrl = ctrl->in(0)->in(0); - } - if (prev_proj != post_loop_entry) { - _igvn.replace_input_of(post_loop_head, LoopNode::EntryControl, prev_proj); - set_idom(post_loop_head, prev_proj, dom_depth(post_loop_head)); - } -} - void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, const uint first_node_index_in_cloned_loop_body, const Node_List& old_new) { const NodeInOriginalLoopBody node_in_original_loop_body(first_node_index_in_cloned_loop_body, old_new); - create_assertion_predicates_at_loop(peeled_loop_head, remaining_loop_head, node_in_original_loop_body); + create_assertion_predicates_at_loop(peeled_loop_head, remaining_loop_head, node_in_original_loop_body, false); +} + +void PhaseIdealLoop::initialize_assertion_predicates_for_main_loop(CountedLoopNode* main_loop_head, + CountedLoopNode* pre_loop_head, + const uint first_node_index_in_cloned_loop_body, + const Node_List& old_new) { + const NodeInOriginalLoopBody node_in_original_loop_body(first_node_index_in_cloned_loop_body, old_new); + create_assertion_predicates_at_loop(pre_loop_head, main_loop_head, node_in_original_loop_body, true); +} + +void PhaseIdealLoop::initialize_assertion_predicates_for_post_loop(CountedLoopNode* main_loop_head, + CountedLoopNode* post_loop_head, + const uint first_node_index_in_cloned_loop_body) { + const NodeInClonedLoopBody node_in_cloned_loop_body(first_node_index_in_cloned_loop_body); + create_assertion_predicates_at_loop(main_loop_head, post_loop_head, node_in_cloned_loop_body, false); } void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, - CountedLoopNode* target_loop_head, - const NodeInLoopBody& _node_in_loop_body) { + CountedLoopNode* target_loop_head, + const NodeInLoopBody& _node_in_loop_body, + const bool clone_template) { Node* init = target_loop_head->init_trip(); Node* stride = target_loop_head->stride(); LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined(); Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl); CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this, - _node_in_loop_body); + _node_in_loop_body, clone_template); Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); PredicateIterator predicate_iterator(source_loop_entry); predicate_iterator.for_each(create_assertion_predicates_visitor); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index fc0f1b4d53c5e..771d504123f5c 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -939,35 +939,28 @@ class PhaseIdealLoop : public PhaseTransform { } #ifdef ASSERT - void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); + static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); #endif - void copy_assertion_predicates_to_main_loop_helper(const PredicateBlock* predicate_block, Node* init, Node* stride, - IdealLoopTree* outer_loop, LoopNode* outer_main_head, - uint dd_main_head, uint idx_before_pre_post, - uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, const Node_List &old_new); - void copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, - LoopNode* outer_main_head, uint dd_main_head, uint idx_before_pre_post, - uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, const Node_List& old_new); - Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, - IdealLoopTree* outer_loop, Node* new_control); public: IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, Node* control); + DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) private: DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);) - DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); - void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, - Node* stride); void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, uint first_node_index_in_cloned_loop_body, const Node_List& old_new); + void initialize_assertion_predicates_for_main_loop(CountedLoopNode* main_loop_head, + CountedLoopNode* pre_loop_head, + uint first_node_index_in_cloned_loop_body, + const Node_List& old_new); + void initialize_assertion_predicates_for_post_loop(CountedLoopNode* main_loop_head, CountedLoopNode* post_loop_head, + uint first_node_index_in_cloned_loop_body); void create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head, - const NodeInLoopBody& _node_in_loop_body); + const NodeInLoopBody& _node_in_loop_body, bool clone_template); void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, Node* bol); void log_loop_tree(); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 54a17376f18e4..ceaf282d575c7 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -162,6 +162,23 @@ bool TemplateAssertionPredicate::is_predicate(Node* node) { return if_node->in(1)->is_OpaqueTemplateAssertionPredicate(); } +// Clone this Template Assertion Predicate and replace the OpaqueLoopInitNode with the provided 'new_opaque_init' node. +IfTrueNode* TemplateAssertionPredicate::clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, + PhaseIdealLoop* phase) const { + assert(PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(_if_node), + "must find OpaqueLoop* nodes for Template Assertion Predicate"); + TemplateAssertionExpression template_assertion_expression(opaque_node()); + OpaqueTemplateAssertionPredicateNode* new_opaque_node = + template_assertion_expression.clone_and_replace_init(new_opaque_init, new_control, phase); + AssertionPredicateIfCreator assertion_predicate_if_creator(phase); + IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, _if_node->Opcode(), + new_opaque_node NOT_PRODUCT(COMMA + _if_node->assertion_predicate_type())); + assert(PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), + "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); + return success_proj; +} + // Initialized Assertion Predicates always have the dedicated OpaqueInitiailizedAssertionPredicate node to identify // them. bool InitializedAssertionPredicate::is_predicate(Node* node) { @@ -743,9 +760,26 @@ void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& t // Only process if we are in the correct Predicate Block. return; } + if (_clone_template) { + _new_control = clone_template_and_replace_init_input(template_assertion_predicate); + } + _new_control = initialize_from_template(template_assertion_predicate); +} + +// Create an Initialized Assertion Predicate from the provided Template Assertion Predicate. +IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template( +const TemplateAssertionPredicate& template_assertion_predicate) const { IfNode* template_head = template_assertion_predicate.head(); IfTrueNode* initialized_predicate = _phase->create_initialized_assertion_predicate(template_head, _init, _stride, _new_control); template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase); - _new_control = initialized_predicate; + return initialized_predicate; +} + +// Clone the provided Template Assertion Predicate and set _init as new input for the OpaqueLoopInitNode. +IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_input( + const TemplateAssertionPredicate& template_assertion_predicate) { + OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _init); + _phase->register_new_node(opaque_init, _new_control); + return template_assertion_predicate.clone_and_replace_init(_new_control, opaque_init, _phase); } diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 5eb0cfc9b3578..f57948f8ab902 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -391,6 +391,10 @@ class TemplateAssertionPredicate : public Predicate { return _if_node->in(0); } + OpaqueTemplateAssertionPredicateNode* opaque_node() const { + return _if_node->in(1)->as_OpaqueTemplateAssertionPredicate(); + } + IfNode* head() const override { return _if_node; } @@ -399,6 +403,7 @@ class TemplateAssertionPredicate : public Predicate { return _success_proj; } + IfTrueNode* clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, PhaseIdealLoop* phase) const; void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); @@ -941,6 +946,22 @@ class NodeInOriginalLoopBody : public NodeInLoopBody { } }; +// This class checks whether a node is in the cloned loop body and not the original one from which the loop was cloned. +class NodeInClonedLoopBody : public NodeInLoopBody { + const uint _first_node_index_in_cloned_loop_body; + + public: + explicit NodeInClonedLoopBody(const uint first_node_index_in_cloned_loop_body) + : _first_node_index_in_cloned_loop_body(first_node_index_in_cloned_loop_body) {} + NONCOPYABLE(NodeInClonedLoopBody); + + // Check if 'node' is a clone. This can easily be achieved by comparing its node index to the first node index + // inside the cloned loop body (all of them are clones). + bool check(Node* node) const override { + return node->_idx >= _first_node_index_in_cloned_loop_body; + } +}; + // Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source // loop. This visitor can be used in combination with a PredicateIterator. class CreateAssertionPredicatesVisitor : public PredicateVisitor { @@ -951,17 +972,22 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { PhaseIdealLoop* const _phase; bool _has_hoisted_check_parse_predicates; const NodeInLoopBody& _node_in_loop_body; + const bool _clone_template; + + IfTrueNode* clone_template_and_replace_init_input(const TemplateAssertionPredicate& template_assertion_predicate); + IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate) const; public: CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase, - const NodeInLoopBody& node_in_loop_body) + const NodeInLoopBody& node_in_loop_body, const bool clone_template) : _init(init), _stride(stride), _old_target_loop_entry(new_control), _new_control(new_control), _phase(phase), _has_hoisted_check_parse_predicates(false), - _node_in_loop_body(node_in_loop_body) {} + _node_in_loop_body(node_in_loop_body), + _clone_template(clone_template) {} NONCOPYABLE(CreateAssertionPredicatesVisitor); using PredicateVisitor::visit;