From 25b207bad1455daa689cdfc9618ddc5f00ec1859 Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Thu, 17 Oct 2024 11:29:54 +0200 Subject: [PATCH] 8342043: Split Opaque4Node into OpaqueTemplateAssertionPredicateNode and OpaqueNotNullNode --- src/hotspot/share/opto/classes.hpp | 3 +- src/hotspot/share/opto/compile.cpp | 2 +- src/hotspot/share/opto/compile.hpp | 5 +- src/hotspot/share/opto/escape.cpp | 6 +- src/hotspot/share/opto/graphKit.cpp | 8 +- src/hotspot/share/opto/loopPredicate.cpp | 13 +-- src/hotspot/share/opto/loopTransform.cpp | 45 ++++---- src/hotspot/share/opto/loopnode.cpp | 10 +- src/hotspot/share/opto/loopnode.hpp | 2 +- src/hotspot/share/opto/loopopts.cpp | 17 ++- src/hotspot/share/opto/macro.cpp | 15 ++- src/hotspot/share/opto/node.cpp | 2 +- src/hotspot/share/opto/node.hpp | 17 +-- src/hotspot/share/opto/opaquenode.cpp | 19 +++- src/hotspot/share/opto/opaquenode.hpp | 37 ++++--- src/hotspot/share/opto/predicates.cpp | 131 ++++++++++++----------- src/hotspot/share/opto/predicates.hpp | 32 +++--- src/hotspot/share/opto/split_if.cpp | 17 +-- 18 files changed, 211 insertions(+), 170 deletions(-) diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index de6d48ebdf99c..7fd488f9d342a 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -270,8 +270,9 @@ macro(Opaque1) macro(OpaqueLoopInit) macro(OpaqueLoopStride) macro(OpaqueZeroTripGuard) -macro(Opaque4) +macro(OpaqueNotNull) macro(OpaqueInitializedAssertionPredicate) +macro(OpaqueTemplateAssertionPredicate) macro(ProfileBoolean) macro(OrI) macro(OrL) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index c6b316e527760..a452d439a540b 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -393,7 +393,7 @@ void Compile::remove_useless_node(Node* dead) { if (dead->is_expensive()) { remove_expensive_node(dead); } - if (dead->Opcode() == Op_Opaque4) { + if (dead->is_OpaqueTemplateAssertionPredicate()) { remove_template_assertion_predicate_opaq(dead); } if (dead->is_ParsePredicate()) { diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 6568828125fe2..05e24bf3f6e1f 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -370,7 +370,8 @@ class Compile : public Phase { GrowableArray _intrinsics; // List of intrinsics. GrowableArray _macro_nodes; // List of nodes which need to be expanded before matching. GrowableArray _parse_predicates; // List of Parse Predicates. - GrowableArray _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates. + // List of OpaqueTemplateAssertionPredicateNode nodes for Template Assertion Predicates. + GrowableArray _template_assertion_predicate_opaqs; GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _unstable_if_traps; // List of ifnodes after IGVN @@ -770,7 +771,7 @@ class Compile : public Phase { void add_template_assertion_predicate_opaq(Node* n) { assert(!_template_assertion_predicate_opaqs.contains(n), - "duplicate entry in template assertion predicate opaque4 list"); + "Duplicate entry in Template Assertion Predicate OpaqueTemplateAssertionPredicate list"); _template_assertion_predicate_opaqs.append(n); } diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index b26480ba9b39a..11c21636024f4 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -574,14 +574,14 @@ bool ConnectionGraph::can_reduce_check_users(Node* n, uint nesting) const { // CmpP/N used by the If controlling the cast. if (use->in(0)->is_IfTrue() || use->in(0)->is_IfFalse()) { Node* iff = use->in(0)->in(0); - // We may have Opaque4 node between If and Bool nodes. - // Bail out in such case - we need to preserve Opaque4 for correct - // processing predicates after loop opts. + // We may have an OpaqueNotNull node between If and Bool nodes. Bail out in such case. bool can_reduce = (iff->Opcode() == Op_If) && iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp(); if (can_reduce) { Node* iff_cmp = iff->in(1)->in(1); int opc = iff_cmp->Opcode(); can_reduce = (opc == Op_CmpP || opc == Op_CmpN) && can_reduce_cmp(n, iff_cmp); + } else { + assert(iff->in(1)->is_OpaqueNotNull(), "must be OpaqueNotNull"); } if (!can_reduce) { #ifndef PRODUCT diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index fe8ca76e318c1..ac74abd7e556b 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -1458,7 +1458,7 @@ Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) { // In that case that data path will die and we need the control path // to become dead as well to keep the graph consistent. So we have to // add a check for null for which one branch can't be taken. It uses -// an Opaque4 node that will cause the check to be removed after loop +// an OpaqueNotNull node that will cause the check to be removed after loop // opts so the test goes away and the compiled code doesn't execute a // useless check. Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) { @@ -1466,9 +1466,9 @@ Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) { return value; } Node* chk = _gvn.transform(new CmpPNode(value, null())); - Node *tst = _gvn.transform(new BoolNode(chk, BoolTest::ne)); - Node* opaq = _gvn.transform(new Opaque4Node(C, tst, intcon(1))); - IfNode *iff = new IfNode(control(), opaq, PROB_MAX, COUNT_UNKNOWN); + Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::ne)); + Node* opaq = _gvn.transform(new OpaqueNotNullNode(C, tst)); + IfNode* iff = new IfNode(control(), opaq, PROB_MAX, COUNT_UNKNOWN); _gvn.set_type(iff, iff->Value(&_gvn)); if (!tst->is_Con()) { record_for_igvn(iff); diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 92f4e3582e96a..d1de9c981012c 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -338,7 +338,8 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree } // Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' -// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections. +// is set, then the OpaqueTemplateAssertionPredicate nodes of the Assertion Predicates are put on the list instead of +// the projections. void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) { ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate(); ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con); @@ -353,10 +354,10 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& } Node* bol = iff->in(1); assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_Opaque4()) { + if (bol->is_OpaqueTemplateAssertionPredicate()) { assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); if (get_opaque) { - // Collect the predicate Opaque4 node. + // Collect the OpaqueTemplateAssertionPredicateNode. list.push(bol); } else { // Collect the predicate projection. @@ -374,11 +375,11 @@ IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfNod IfProjNode* predicate, Deoptimization::DeoptReason reason, ParsePredicateSuccessProj* parse_predicate_proj) { - TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_Opaque4()); - Opaque4Node* cloned_opaque4_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this); + TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate()); + OpaqueTemplateAssertionPredicateNode* cloned_opaque_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this); IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, template_assertion_predicate->Opcode(), false); - _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque4_node); + _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque_node); _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj)); return if_proj; diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 453622b447343..322183ec3a092 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1189,8 +1189,10 @@ bool IdealLoopTree::policy_range_check(PhaseIdealLoop* phase, bool provisional, continue; } if (!bol->is_Bool()) { - assert(bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate(), - "Opaque node of non-null-check or of Initialized Assertion Predicate"); + assert(bol->is_OpaqueNotNull() || + bol->is_OpaqueTemplateAssertionPredicate() || + bol->is_OpaqueInitializedAssertionPredicate(), + "Opaque node of a non-null-check or an Assertion Predicate"); continue; } if (bol->as_Bool()->_test._test == BoolTest::ne) { @@ -1359,7 +1361,7 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica break; Node* bol = iff->in(1); assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_Opaque4()) { + if (bol->is_OpaqueTemplateAssertionPredicate()) { // Clone the Assertion Predicate twice and initialize one with the initial // value of the loop induction variable. Leave the other predicate // to be initialized when increasing the stride during loop unrolling. @@ -1399,11 +1401,11 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica } } +#ifdef ASSERT bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { uint init; uint stride; count_opaque_loop_nodes(iff->in(1)->in(1), init, stride); -#ifdef ASSERT ResourceMark rm; Unique_Node_List wq; wq.clear(); @@ -1429,10 +1431,10 @@ bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { } } assert(init == verif_init && stride == verif_stride, "missed opaque node"); -#endif assert(stride == 0 || init != 0, "init should be there every time stride is"); return init != 0; } +#endif // ASSERT void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) { init = 0; @@ -1475,14 +1477,15 @@ IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* templ // Clone the Template Assertion Predicate and set a new OpaqueLoopInitNode to create a new Template Assertion Predicate. // This is done when creating a new Template Assertion Predicate for the main loop which requires a new init node. -// We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts, -// these are never executed. We therefore insert a Halt node instead of an uncommon trap. +// We keep the OpaqueTemplateAssertionPredicate node since it's still a template. Since the templates are eventually +// removed after loop opts, these are never executed. We therefore insert a Halt node instead of an uncommon trap. Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, Node* new_control) { assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate"); - TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4()); + TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_OpaqueTemplateAssertionPredicate()); assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); - Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this); + OpaqueTemplateAssertionPredicateNode* new_opaque_node = + template_assertion_expression.clone_and_replace_init(new_init, control, this); AssertionPredicateIfCreator assertion_predicate_if_creator(this); IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, iff->Opcode(), new_opaque_node @@ -1922,18 +1925,13 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo break; } Node* bol = iff->in(1); - if (bol->is_Opaque4()) { - if (assertion_predicate_has_loop_opaque_node(iff)) { - // This is a Template Assertion Predicate for the initial or last access. - // Create an Initialized Assertion Predicates for it accordingly: - // - For the initial access a[init] (same as before) - // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride) - prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj); - } else { - // Ignore Opaque4 from a non-null-check for an intrinsic or unsafe access. This could happen when we maximally - // unroll a non-main loop with such an If with an Opaque4 node directly above the loop entry. - assert(!loop_head->is_main_loop(), "Opaque4 node from a non-null check - should not be at main loop"); - } + if (bol->is_OpaqueTemplateAssertionPredicate()) { + assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); + // This is a Template Assertion Predicate for the initial or last access. + // Create an Initialized Assertion Predicates for it accordingly: + // - For the initial access a[init] (same as before) + // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride) + prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj); } else if (bol->is_OpaqueInitializedAssertionPredicate()) { // This is one of the two Initialized Assertion Predicates: // - For the initial access a[init] @@ -1941,6 +1939,7 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo // We could keep the one for the initial access but we do not know which one we currently have here. Just kill both. _igvn.replace_input_of(iff, 1, _igvn.intcon(1)); } + assert(!bol->is_OpaqueNotNull() || !loop_head->is_main_loop(), "OpaqueNotNull should not be at main loop"); entry = entry->in(0)->in(0); } if (prev_proj != ctrl) { @@ -1967,7 +1966,7 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_ if (!proj->unique_ctrl_out()->is_Halt()) { break; } - if (iff->in(1)->is_Opaque4()) { + if (iff->in(1)->is_OpaqueTemplateAssertionPredicate()) { // Initialize from Template Assertion Predicate. prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj); } @@ -2000,7 +1999,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const Predi } Node* bol = iff->in(1); assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_Opaque4()) { + if (bol->is_OpaqueTemplateAssertionPredicate()) { // Initialize from Template Assertion Predicate. input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj); diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 90b15381f823b..c715d03efc021 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4381,7 +4381,8 @@ void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() { // Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by -// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN. +// replacing the OpaqueTemplateAssertionPredicate node of the If node with true. These nodes will be removed during the +// next round of IGVN. void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() { Unique_Node_List useful_predicates; if (C->has_loops()) { @@ -4422,9 +4423,10 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) { for (int i = C->template_assertion_predicate_count(); i > 0; i--) { - Opaque4Node* opaque4_node = C->template_assertion_predicate_opaq_node(i - 1)->as_Opaque4(); - if (!useful_predicates.member(opaque4_node)) { // not in the useful list - _igvn.replace_node(opaque4_node, opaque4_node->in(2)); + OpaqueTemplateAssertionPredicateNode* opaque_node = + C->template_assertion_predicate_opaq_node(i - 1)->as_OpaqueTemplateAssertionPredicate(); + if (!useful_predicates.member(opaque_node)) { // not in the useful list + _igvn.replace_node(opaque_node, _igvn.intcon(1)); } } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ef27eb652f744..7b6debe525b45 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -955,7 +955,7 @@ class PhaseIdealLoop : public PhaseTransform { IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, Node* control); static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); - static bool assertion_predicate_has_loop_opaque_node(IfNode* iff); + DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 654262d21cb71..941cf41d28a13 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -787,10 +787,9 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { }//for Node* bol = iff->in(1); assert(!bol->is_OpaqueInitializedAssertionPredicate(), "Initialized Assertion Predicates cannot form a diamond with Halt"); - if (bol->is_Opaque4()) { - // Ignore Template Assertion Predicates with Opaque4 nodes. - assert(assertion_predicate_has_loop_opaque_node(iff), - "must be Template Assertion Predicate, non-null-check with Opaque4 cannot form a diamond with Halt"); + if (bol->is_OpaqueTemplateAssertionPredicate()) { + // Ignore Template Assertion Predicates with OpaqueTemplateAssertionPredicate nodes. + assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); return nullptr; } assert(bol->Opcode() == Op_Bool, "Unexpected node"); @@ -1702,7 +1701,7 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { !n->is_Proj() && !n->is_MergeMem() && !n->is_CMove() && - !n->is_Opaque4() && + !n->is_OpaqueNotNull() && !n->is_OpaqueInitializedAssertionPredicate() && !n->is_Type()) { Node *n_ctrl = get_ctrl(n); @@ -2020,14 +2019,14 @@ Node* PhaseIdealLoop::clone_iff(PhiNode* phi) { if (b->is_Phi()) { _igvn.replace_input_of(phi, i, clone_iff(b->as_Phi())); } else { - assert(b->is_Bool() || b->is_Opaque4() || b->is_OpaqueInitializedAssertionPredicate(), - "bool, non-null check with Opaque4 node or Initialized Assertion Predicate with its Opaque node"); + assert(b->is_Bool() || b->is_OpaqueNotNull() || b->is_OpaqueInitializedAssertionPredicate(), + "bool, non-null check with OpaqueNotNull or Initialized Assertion Predicate with its Opaque node"); } } Node* n = phi->in(1); Node* sample_opaque = nullptr; Node *sample_bool = nullptr; - if (n->is_Opaque4() || n->is_OpaqueInitializedAssertionPredicate()) { + if (n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate()) { sample_opaque = n; sample_bool = n->in(1); assert(sample_bool->is_Bool(), "wrong type"); @@ -2201,7 +2200,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new, // For example, it is unexpected that there is a Phi between an // AllocateArray node and its ValidLengthTest input that could cause // split if to break. - if (use->is_If() || use->is_CMove() || use->is_Opaque4() || use->is_OpaqueInitializedAssertionPredicate() || + if (use->is_If() || use->is_CMove() || use->is_OpaqueNotNull() || use->is_OpaqueInitializedAssertionPredicate() || (use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) { // Since this code is highly unlikely, we lazily build the worklist // of such Nodes to go split. diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index de804457a262d..604501c861974 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -2428,7 +2428,7 @@ void PhaseMacroExpand::eliminate_macro_nodes() { break; default: assert(n->Opcode() == Op_LoopLimit || - n->is_Opaque4() || + n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate() || n->Opcode() == Op_MaxL || n->Opcode() == Op_MinL || @@ -2480,17 +2480,14 @@ bool PhaseMacroExpand::expand_macro_nodes() { } else if (n->is_Opaque1()) { _igvn.replace_node(n, n->in(1)); success = true; - } else if (n->is_Opaque4()) { - // With Opaque4 nodes, the expectation is that the test of input 1 - // is always equal to the constant value of input 2. So we can - // remove the Opaque4 and replace it by input 2. In debug builds, - // leave the non constant test in instead to sanity check that it - // never fails (if it does, that subgraph was constructed so, at - // runtime, a Halt node is executed). + } else if (n->is_OpaqueNotNull()) { + // Tests with OpaqueNotNull nodes are implicitly known to be true. Replace the node with true. In debug builds, + // we leave the test in the graph to have an additional sanity check at runtime. If the test fails, we will + // execute a Halt node. #ifdef ASSERT _igvn.replace_node(n, n->in(1)); #else - _igvn.replace_node(n, n->in(2)); + _igvn.replace_node(n, _igvn.intcon(1)); #endif success = true; } else if (n->is_OpaqueInitializedAssertionPredicate()) { diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 2e586de33a3d8..8dbf09f7d4fd9 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -611,7 +611,7 @@ void Node::destruct(PhaseValues* phase) { if (is_expensive()) { compile->remove_expensive_node(this); } - if (is_Opaque4()) { + if (is_OpaqueTemplateAssertionPredicate()) { compile->remove_template_assertion_predicate_opaq(this); } if (is_ParsePredicate()) { diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 07a623e2f936d..3764f92234210 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -138,7 +138,8 @@ class NeverBranchNode; class Opaque1Node; class OpaqueLoopInitNode; class OpaqueLoopStrideNode; -class Opaque4Node; +class OpaqueNotNullNode; +class OpaqueTemplateAssertionPredicateNode; class OpaqueInitializedAssertionPredicateNode; class OuterStripMinedLoopNode; class OuterStripMinedLoopEndNode; @@ -796,11 +797,12 @@ class Node { DEFINE_CLASS_ID(Opaque1, Node, 16) DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0) DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1) - DEFINE_CLASS_ID(Opaque4, Node, 17) - DEFINE_CLASS_ID(OpaqueInitializedAssertionPredicate, Node, 18) - DEFINE_CLASS_ID(Move, Node, 19) - DEFINE_CLASS_ID(LShift, Node, 20) - DEFINE_CLASS_ID(Neg, Node, 21) + DEFINE_CLASS_ID(OpaqueNotNull, Node, 17) + DEFINE_CLASS_ID(OpaqueTemplateAssertionPredicate, Node, 18) + DEFINE_CLASS_ID(OpaqueInitializedAssertionPredicate, Node, 19) + DEFINE_CLASS_ID(Move, Node, 20) + DEFINE_CLASS_ID(LShift, Node, 21) + DEFINE_CLASS_ID(Neg, Node, 22) _max_classes = ClassMask_Neg }; @@ -970,7 +972,8 @@ class Node { DEFINE_CLASS_QUERY(NegV) DEFINE_CLASS_QUERY(NeverBranch) DEFINE_CLASS_QUERY(Opaque1) - DEFINE_CLASS_QUERY(Opaque4) + DEFINE_CLASS_QUERY(OpaqueNotNull) + DEFINE_CLASS_QUERY(OpaqueTemplateAssertionPredicate) DEFINE_CLASS_QUERY(OpaqueInitializedAssertionPredicate) DEFINE_CLASS_QUERY(OpaqueLoopInit) DEFINE_CLASS_QUERY(OpaqueLoopStride) diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index 14fd0d5f1a7a9..93a96a5166c66 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -82,7 +82,24 @@ IfNode* OpaqueZeroTripGuardNode::if_node() const { return iff->as_If(); } -const Type* Opaque4Node::Value(PhaseGVN* phase) const { +const Type* OpaqueNotNullNode::Value(PhaseGVN* phase) const { + return phase->type(in(1)); +} + +Node* OpaqueTemplateAssertionPredicateNode::Identity(PhaseGVN* phase) { + if (phase->C->post_loop_opts_phase()) { + // Template Assertion Predicates only serve as templates to create Initialized Assertion Predicates when splitting + // a loop during loop opts. They are not used anymore once loop opts are over and can then be removed. They feed + // into a BoolNode and can thus be replaced by true to let the Template Assertion Predicate fold away (the success + // path is always the true path). + return phase->intcon(1); + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } + return this; +} + +const Type* OpaqueTemplateAssertionPredicateNode::Value(PhaseGVN* phase) const { return phase->type(in(1)); } diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index 9c775408cc0cd..ac8a346ebaa0b 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -91,18 +91,16 @@ class OpaqueZeroTripGuardNode : public Opaque1Node { IfNode* if_node() const; }; -// Input 1 is a check that we know implicitly is always true or false -// but the compiler has no way to prove. If during optimizations, that -// check becomes true or false, the Opaque4 node is replaced by that -// constant true or false. Input 2 is the constant value we know the -// test takes. After loop optimizations, we replace input 1 by input 2 -// so the control that depends on that test can be removed and there's -// no overhead at runtime. Used for instance by -// GraphKit::must_be_not_null(). -class Opaque4Node : public Node { - public: - Opaque4Node(Compile* C, Node* tst, Node* final_tst) : Node(nullptr, tst, final_tst) { - init_class_id(Class_Opaque4); +// This node is used in the context of intrinsics. We sometimes implicitly know that an object is non-null even though +// the compiler cannot prove it. We therefore add a corresponding cast to propagate this implicit knowledge. However, +// this cast could become top during optimizations (input to cast becomes null) and the data path is folded. To ensure +// that the control path is also properly folded, we insert an If node with such a OpaqueNotNullNode as condition. +// During macro expansion, we replace the OpaqueNotNullNode with true such that the actually unneeded check is folded +// and does not end up in the emitted code. Also see GraphKit::must_be_not_null() for more details. +class OpaqueNotNullNode : public Node { + public: + OpaqueNotNullNode(Compile* C, Node* tst) : Node(nullptr, tst) { + init_class_id(Class_OpaqueNotNull); init_flags(Flag_is_macro); C->add_macro_node(this); } @@ -112,6 +110,21 @@ class Opaque4Node : public Node { virtual const Type* bottom_type() const { return TypeInt::BOOL; } }; +// This node is used for Template Assertion Predicate BoolNodes. A Template Assertion Predicate is always removed +// after loop opts and thus is never converted to actual code. In the post loop opts IGVN phase, the +// OpaqueTemplateAssertionPredicateNode is replaced by true in order to fold the Template Assertion Predicate away. +class OpaqueTemplateAssertionPredicateNode : public Node { + public: + OpaqueTemplateAssertionPredicateNode(BoolNode* bol) : Node(nullptr, bol) { + init_class_id(Class_OpaqueTemplateAssertionPredicate); + } + + virtual int Opcode() const; + virtual Node* Identity(PhaseGVN* phase); + virtual const Type* Value(PhaseGVN* phase) const; + virtual const Type* bottom_type() const { return TypeInt::BOOL; } +}; + // This node is used for Initialized Assertion Predicate BoolNodes. Initialized Assertion Predicates must always evaluate // to true. Therefore, we get rid of them in product builds during macro expansion as they are useless. In debug builds // we keep them as additional verification code (i.e. removing this node and use the BoolNode input instead). diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 153a2bcd44223..6c6f166594beb 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -55,12 +55,12 @@ bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) { return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj); } -// Check if the If node of `predicate_proj` has an Opaque4 (Template Assertion Predicate) or an -// OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input. +// Check if the If node of `predicate_proj` has an OpaqueTemplateAssertionPredicate (Template Assertion Predicate) or +// an OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input. bool AssertionPredicateWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { IfNode* iff = predicate_proj->in(0)->as_If(); Node* bol = iff->in(1); - return bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate(); + return bol->is_OpaqueTemplateAssertionPredicate() || bol->is_OpaqueInitializedAssertionPredicate(); } // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. @@ -137,22 +137,19 @@ bool RuntimePredicate::is_predicate(Node* node, Deoptimization::DeoptReason deop return RegularPredicateWithUCT::is_predicate(node, deopt_reason); } -// A Template Assertion Predicate has an If/RangeCheckNode and either an UCT or a halt node depending on where it -// was created. +// Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them. bool TemplateAssertionPredicate::is_predicate(Node* node) { if (!may_be_assertion_predicate_if(node)) { return false; } IfNode* if_node = node->in(0)->as_If(); - if (if_node->in(1)->is_Opaque4()) { - return RegularPredicateWithUCT::has_valid_uncommon_trap(node) || AssertionPredicateWithHalt::has_halt(node); - } - return false; + return if_node->in(1)->is_OpaqueTemplateAssertionPredicate(); } -// Initialized Assertion Predicates always have the dedicated opaque node and a halt node. +// Initialized Assertion Predicates always have the dedicated OpaqueInitiailizedAssertionPredicate node to identify +// them. bool InitializedAssertionPredicate::is_predicate(Node* node) { - if (!AssertionPredicateWithHalt::is_predicate(node)) { + if (!may_be_assertion_predicate_if(node)) { return false; } IfNode* if_node = node->in(0)->as_If(); @@ -238,27 +235,27 @@ class ReplaceInitAndStrideStrategy : public TransformStrategyForOpaqueLoopNodes } }; -// Creates an identical clone of this Template Assertion Expression (i.e.cloning all nodes from the Opaque4Node to and -// including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the same graph structure as found for this -// Template Assertion Expression. The cloned nodes get 'new_ctrl' as ctrl. There is no other update done for the cloned -// nodes. Return the newly cloned Opaque4Node. -Opaque4Node* TemplateAssertionExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) { +// Creates an identical clone of this Template Assertion Expression (i.e.cloning all nodes from the +// OpaqueTemplateAssertionPredicate to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the +// same graph structure as found for this Template Assertion Expression. The cloned nodes get 'new_ctrl' as ctrl. There +// is no other update done for the cloned nodes. Return the newly cloned OpaqueTemplateAssertionPredicate. +OpaqueTemplateAssertionPredicateNode* TemplateAssertionExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) { CloneStrategy clone_init_and_stride_strategy(phase, new_ctrl); return clone(clone_init_and_stride_strategy, new_ctrl, phase); } // Same as clone() but instead of cloning the OpaqueLoopInitNode, we replace it with the provided 'new_init' node. -Opaque4Node* TemplateAssertionExpression::clone_and_replace_init(Node* new_init, Node* new_ctrl, - PhaseIdealLoop* phase) { +OpaqueTemplateAssertionPredicateNode* +TemplateAssertionExpression::clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase) { ReplaceInitAndCloneStrideStrategy replace_init_and_clone_stride_strategy(new_init, new_ctrl, phase); return clone(replace_init_and_clone_stride_strategy, new_ctrl, phase); } // Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided // 'new_init' and 'new_stride' nodes, respectively. -Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_control, Node* new_init, - Node* new_stride, - PhaseIdealLoop* phase) { +OpaqueTemplateAssertionPredicateNode* +TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride, + PhaseIdealLoop* phase) { ReplaceInitAndStrideStrategy replace_init_and_stride_strategy(new_init, new_stride); return clone(replace_init_and_stride_strategy, new_control, phase); } @@ -341,20 +338,21 @@ class DataNodesOnPathsToTargets : public StackObj { }; // Clones this Template Assertion Expression and applies the given strategy to transform the OpaqueLoop* nodes. -Opaque4Node* TemplateAssertionExpression::clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, - Node* new_ctrl, PhaseIdealLoop* phase) { +OpaqueTemplateAssertionPredicateNode* +TemplateAssertionExpression::clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* new_ctrl, + PhaseIdealLoop* phase) { ResourceMark rm; auto is_opaque_loop_node = [](const Node* node) { return node->is_Opaque1(); }; DataNodesOnPathsToTargets data_nodes_on_path_to_targets(TemplateAssertionExpressionNode::is_maybe_in_expression, is_opaque_loop_node); - const Unique_Node_List& collected_nodes = data_nodes_on_path_to_targets.collect(_opaque4_node); + const Unique_Node_List& collected_nodes = data_nodes_on_path_to_targets.collect(_opaque_node); DataNodeGraph data_node_graph(collected_nodes, phase); const OrigToNewHashtable& orig_to_new = data_node_graph.clone_with_opaque_loop_transform_strategy(transform_strategy, new_ctrl); - assert(orig_to_new.contains(_opaque4_node), "must exist"); - Node* opaque4_clone = *orig_to_new.get(_opaque4_node); - return opaque4_clone->as_Opaque4(); + assert(orig_to_new.contains(_opaque_node), "must exist"); + Node* opaque_node_clone = *orig_to_new.get(_opaque_node); + return opaque_node_clone->as_OpaqueTemplateAssertionPredicate(); } // Check if this node belongs a Template Assertion Expression (including OpaqueLoop* nodes). @@ -376,7 +374,7 @@ bool TemplateAssertionExpressionNode::is_in_expression(Node* node) { } bool TemplateAssertionExpressionNode::is_template_assertion_predicate(Node* node) { - return node->is_If() && node->in(1)->is_Opaque4(); + return node->is_If() && node->in(1)->is_OpaqueTemplateAssertionPredicate(); } // This class creates the Assertion Predicate expression to be used for a Template or Initialized Assertion Predicate. @@ -398,18 +396,18 @@ class AssertionPredicateExpressionCreator : public StackObj { _range(range), _upper((_stride > 0) != (_scale > 0)) {} // Make sure rc_predicate() chooses the "scale*init + offset" case. - // Create the expression for a Template Assertion Predicate with an Opaque4 node. - Opaque4Node* create_for_template(Node* new_control, Node* operand, bool& does_overflow) const { + // Create the expression for a Template Assertion Predicate with an OpaqueTemplateAssertionPredicate node. + OpaqueTemplateAssertionPredicateNode* create_for_template(Node* new_control, Node* operand, bool& does_overflow) const { BoolNode* bool_for_expression = _phase->rc_predicate(new_control, _scale, _offset, operand, nullptr, _stride, _range, _upper, does_overflow); - return create_opaque4_node(new_control, bool_for_expression); + return create_opaque_node(new_control, bool_for_expression); } private: - Opaque4Node* create_opaque4_node(Node* new_control, BoolNode* bool_for_expression) const { - Compile* C = _phase->C; - Opaque4Node* new_expression = new Opaque4Node(C, bool_for_expression, _phase->igvn().intcon(1)); - C->add_template_assertion_predicate_opaq(new_expression); + OpaqueTemplateAssertionPredicateNode* create_opaque_node(Node* new_control, BoolNode* bool_for_expression) const { + OpaqueTemplateAssertionPredicateNode* new_expression = + new OpaqueTemplateAssertionPredicateNode(bool_for_expression); + _phase->C->add_template_assertion_predicate_opaq(new_expression); _phase->register_new_node(new_expression, new_control); return new_expression; } @@ -463,8 +461,8 @@ IfTrueNode* AssertionPredicateIfCreator::create_for_initialized(Node* new_contro IfTrueNode* AssertionPredicateIfCreator::create(Node* new_control, const int if_opcode, Node* assertion_expression, const char* halt_message NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)) { - assert(assertion_expression->is_Opaque4() || assertion_expression->is_OpaqueInitializedAssertionPredicate(), - "not a valid assertion expression"); + assert(assertion_expression->is_OpaqueTemplateAssertionPredicate() || + assertion_expression->is_OpaqueInitializedAssertionPredicate(), "not a valid assertion expression"); IdealLoopTree* loop = _phase->get_loop(new_control); IfNode* if_node = create_if_node(new_control, if_opcode, assertion_expression, loop NOT_PRODUCT(COMMA assertion_predicate_type)); @@ -517,8 +515,8 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_with_uncommon_trap( const Deoptimization::DeoptReason deopt_reason, const int if_opcode) { OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control); bool does_overflow; - Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init, - does_overflow); + OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression = + create_for_init_value(new_control, opaque_init, does_overflow); IfTrueNode* template_predicate_success_proj = create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj, deopt_reason, if_opcode, does_overflow @@ -536,15 +534,17 @@ OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* return opaque_init; } -Opaque4Node* TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, - bool& does_overflow) const { +OpaqueTemplateAssertionPredicateNode* +TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const { AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase); return expression_creator.create_for_template(new_control, opaque_init, does_overflow); } IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap( - Opaque4Node* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj, - const Deoptimization::DeoptReason deopt_reason, const int if_opcode, const bool does_overflow + OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, + ParsePredicateSuccessProj* parse_predicate_success_proj, const Deoptimization::DeoptReason deopt_reason, + const int if_opcode, const bool does_overflow NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { IfTrueNode* success_proj = _phase->create_new_if_for_predicate(parse_predicate_success_proj, nullptr, deopt_reason, does_overflow ? Op_If : if_opcode, false @@ -553,8 +553,9 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap return success_proj; } -Opaque4Node* TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, - bool& does_overflow) const { +OpaqueTemplateAssertionPredicateNode* +TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const { Node* last_value = create_last_value(new_control, opaque_init); AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase); return expression_creator.create_for_template(new_control, last_value, does_overflow); @@ -575,7 +576,7 @@ Node* TemplateAssertionPredicateCreator::create_last_value(Node* new_control, Op } IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt( - Node* new_control, Opaque4Node* template_assertion_predicate_expression, bool does_overflow + Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, bool does_overflow NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); return assertion_predicate_if_creator.create_for_template(new_control, does_overflow ? Op_If : Op_RangeCheck, @@ -588,8 +589,8 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt( IfTrueNode* TemplateAssertionPredicateCreator::create_with_halt(Node* new_control) { OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control); bool does_overflow; - Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init, - does_overflow); + OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression = + create_for_init_value(new_control, opaque_init, does_overflow); IfTrueNode* template_predicate_success_proj = create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); @@ -604,23 +605,23 @@ InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(Phase // Create an Initialized Assertion Predicate from the provided template_assertion_predicate at 'new_control'. // We clone the Template Assertion Expression and replace: -// - Opaque4 with OpaqueInitializedAssertionPredicate +// - OpaqueTemplateAssertionPredicateNode with OpaqueInitializedAssertionPredicate // - OpaqueLoop*Nodes with new_init and _ew_stride, respectively. // // / init stride // | | | -// | OpaqueLoopInitNode OpaqueLoopStrideNode / new_init new_stride -// Template | \ / | \ / -// Assertion | ... Assertion | ... -// Expression | | Expression | | -// | Bool | new Bool -// | | | | -// \ Opaque4 ======> new_control \ OpaqueInitializedAssertionPredicate -// | \ / -// If new If -// / \ / \ -// success fail path new success new Halt -// proj (Halt or UCT) proj +// | OpaqueLoopInitNode OpaqueLoopStrideNode / new_init new_stride +// Template | \ / | \ / +// Assertion | ... Assertion | ... +// Expression | | Expression | | +// | Bool | new Bool +// | | | | +// \ OpaqueTemplateAssertionPredicate ===> new_control \ OpaqueInitializedAssertionPredicate +// | \ / +// If new If +// / \ / \ +// success fail path new success new Halt +// proj (Halt or UCT) proj // IfTrueNode* InitializedAssertionPredicateCreator::create_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init, @@ -658,11 +659,11 @@ OpaqueInitializedAssertionPredicateNode* InitializedAssertionPredicateCreator::create_assertion_expression_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init, Node* new_stride) { - Opaque4Node* template_opaque = template_assertion_predicate->in(1)->as_Opaque4(); + OpaqueTemplateAssertionPredicateNode* template_opaque = + template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate(); TemplateAssertionExpression template_assertion_expression(template_opaque); - Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(new_control, new_init, - new_stride, - _phase); + OpaqueTemplateAssertionPredicateNode* tmp_opaque = + template_assertion_expression.clone_and_replace_init_and_stride(new_control, new_init, new_stride, _phase); OpaqueInitializedAssertionPredicateNode* assertion_expression = new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C); _phase->register_new_node(assertion_expression, new_control); diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 4df5667303552..2ce3a622266ab 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -82,8 +82,8 @@ class TemplateAssertionPredicate; * actually dead. Assertion Predicates come to the rescue to fold such seemingly dead sub loops * away to avoid a broken graph. Assertion Predicates are left in the graph as a sanity checks in * debug builds (they must never fail at runtime) while they are being removed in product builds. - * We use special Opaque4 nodes to block some optimizations and replace the Assertion Predicates - * later in product builds. + * We use special OpaqueTemplateAssertionPredicateNode nodes to block some optimizations and replace + * the Assertion Predicates later in product builds. * * There are two kinds of Assertion Predicates: * - Template Assertion Predicate: A template for an Assertion Predicate that uses OpaqueLoop* @@ -431,21 +431,22 @@ class TransformStrategyForOpaqueLoopNodes : public StackObj { virtual Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) const = 0; }; -// A Template Assertion Predicate represents the Opaque4Node for the initial value or the last value of a -// Template Assertion Predicate and all the nodes up to and including the OpaqueLoop* nodes. +// A Template Assertion Predicate represents the OpaqueTemplateAssertionPredicateNode for the initial value or the last +// value of a Template Assertion Predicate and all the nodes up to and including the OpaqueLoop* nodes. class TemplateAssertionExpression : public StackObj { - Opaque4Node* _opaque4_node; + OpaqueTemplateAssertionPredicateNode* _opaque_node; public: - explicit TemplateAssertionExpression(Opaque4Node* opaque4_node) : _opaque4_node(opaque4_node) {} + explicit TemplateAssertionExpression(OpaqueTemplateAssertionPredicateNode* opaque_node) : _opaque_node(opaque_node) {} private: - Opaque4Node* clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* new_ctrl, PhaseIdealLoop* phase); + OpaqueTemplateAssertionPredicateNode* clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* new_ctrl, + PhaseIdealLoop* phase); public: - Opaque4Node* clone(Node* new_ctrl, PhaseIdealLoop* phase); - Opaque4Node* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase); - Opaque4Node* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride, + OpaqueTemplateAssertionPredicateNode* clone(Node* new_ctrl, PhaseIdealLoop* phase); + OpaqueTemplateAssertionPredicateNode* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase); + OpaqueTemplateAssertionPredicateNode* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride, PhaseIdealLoop* phase); }; @@ -554,15 +555,18 @@ class TemplateAssertionPredicateCreator : public StackObj { PhaseIdealLoop* const _phase; OpaqueLoopInitNode* create_opaque_init(Node* new_control); - Opaque4Node* create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const; - Opaque4Node* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const; + OpaqueTemplateAssertionPredicateNode* create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const; + OpaqueTemplateAssertionPredicateNode* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const; Node* create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const; - IfTrueNode* create_if_node_with_uncommon_trap(Opaque4Node* template_assertion_predicate_expression, + IfTrueNode* create_if_node_with_uncommon_trap(OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj, Deoptimization::DeoptReason deopt_reason, int if_opcode, bool does_overflow NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); - IfTrueNode* create_if_node_with_halt(Node* new_control, Opaque4Node* template_assertion_predicate_expression, + IfTrueNode* create_if_node_with_halt(Node* new_control, + OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, bool does_overflow NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp index 1eff44ab7834f..787c278239e41 100644 --- a/src/hotspot/share/opto/split_if.cpp +++ b/src/hotspot/share/opto/split_if.cpp @@ -322,7 +322,8 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) assert( bol->is_Bool(), "" ); if (bol->outcnt() == 1) { Node* use = bol->unique_out(); - if (use->is_Opaque4() || use->is_OpaqueInitializedAssertionPredicate()) { + if (use->is_OpaqueNotNull() || use->is_OpaqueTemplateAssertionPredicate() || + use->is_OpaqueInitializedAssertionPredicate()) { if (use->outcnt() == 1) { Node* iff = use->unique_out(); assert(iff->is_If(), "unexpected node type"); @@ -351,8 +352,9 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) #endif for (DUIterator j = bol->outs(); bol->has_out(j); j++) { Node* u = bol->out(j); - // Uses are either IfNodes, CMoves, Opaque4, or OpaqueInitializedAssertionPredicates - if (u->is_Opaque4() || u->is_OpaqueInitializedAssertionPredicate()) { + // Uses are either IfNodes, CMoves, OpaqueNotNull, or Opaque*AssertionPredicates + if (u->is_OpaqueNotNull() || u->is_OpaqueTemplateAssertionPredicate() || + u->is_OpaqueInitializedAssertionPredicate()) { assert(u->in(1) == bol, "bad input"); for (DUIterator_Last kmin, k = u->last_outs(kmin); k >= kmin; --k) { Node* iff = u->last_out(k); @@ -421,11 +423,12 @@ void PhaseIdealLoop::clone_template_assertion_expression_down(Node* node) { TemplateAssertionExpressionNode template_assertion_expression_node(node); auto clone_expression = [&](IfNode* template_assertion_predicate) { - Opaque4Node* opaque4_node = template_assertion_predicate->in(1)->as_Opaque4(); - TemplateAssertionExpression template_assertion_expression(opaque4_node); + OpaqueTemplateAssertionPredicateNode* opaque_node = + template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate(); + TemplateAssertionExpression template_assertion_expression(opaque_node); Node* new_ctrl = template_assertion_predicate->in(0); - Opaque4Node* cloned_opaque4_node = template_assertion_expression.clone(new_ctrl, this); - igvn().replace_input_of(template_assertion_predicate, 1, cloned_opaque4_node); + OpaqueTemplateAssertionPredicateNode* cloned_opaque_node = template_assertion_expression.clone(new_ctrl, this); + igvn().replace_input_of(template_assertion_predicate, 1, cloned_opaque_node); }; template_assertion_expression_node.for_each_template_assertion_predicate(clone_expression); }