From 1be0a73cf447ac6f369a53ffe9e60d5d2d16cb9a Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Tue, 1 Oct 2024 13:24:09 +0200 Subject: [PATCH] 8341328: Refactor Assertion Predicate creation from scratch into separate classes --- src/hotspot/share/opto/ifnode.cpp | 3 + src/hotspot/share/opto/loopPredicate.cpp | 81 ++---- src/hotspot/share/opto/loopTransform.cpp | 122 +++------ src/hotspot/share/opto/loopnode.hpp | 15 +- src/hotspot/share/opto/predicates.cpp | 325 +++++++++++++++++++---- src/hotspot/share/opto/predicates.hpp | 92 +++++-- 6 files changed, 410 insertions(+), 228 deletions(-) diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index 4313b2cf907a9..093cafd1574db 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -1849,6 +1849,9 @@ void IfNode::dump_spec(outputStream* st) const { case AssertionPredicateType::LastValue: st->print("#Last Value Assertion Predicate "); break; + case AssertionPredicateType::FinalIv: + st->print("#Final IV Assertion Predicate "); + break; case AssertionPredicateType::None: // No Assertion Predicate break; diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 5e585a406f20c..92f4e3582e96a 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -1153,7 +1153,8 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, - ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) { + ConNode* zero, Invariance& invar, + Deoptimization::DeoptReason deopt_reason) { // Following are changed to nonnull when a predicate can be hoisted IfNode* iff = if_success_proj->in(0)->as_If(); Node* test = iff->in(1); @@ -1165,7 +1166,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod if (invar.is_invariant(bol)) { C->print_method(PHASE_BEFORE_LOOP_PREDICATION_IC, 4, iff); // Invariant test - IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, + IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason, iff->Opcode()); Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0); BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool(); @@ -1206,9 +1207,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod const Node* cmp = bol->in(1)->as_Cmp(); Node* idx = cmp->in(1); assert(!invar.is_invariant(idx), "index is variant"); - Node* rng = cmp->in(2); - assert(rng->Opcode() == Op_LoadRange || iff->is_RangeCheck() || _igvn.type(rng)->is_int()->_lo >= 0, "must be"); - assert(invar.is_invariant(rng), "range must be invariant"); + Node* range = cmp->in(2); + assert(range->Opcode() == Op_LoadRange || iff->is_RangeCheck() || _igvn.type(range)->is_int()->_lo >= 0, "must be"); + assert(invar.is_invariant(range), "range must be invariant"); int scale = 1; Node* offset = zero; bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset); @@ -1237,7 +1238,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // late schedule will place invariant things in the loop. ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); Node* ctrl = parse_predicate->in(0); - rng = invar.clone(rng, ctrl); + range = invar.clone(range, ctrl); if (offset && offset != zero) { assert(invar.is_invariant(offset), "offset must be loop invariant"); offset = invar.clone(offset, ctrl); @@ -1245,10 +1246,10 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // If predicate expressions may overflow in the integer range, longs are used. bool overflow = false; // Test the lower bound - BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false, overflow); + BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, range, false, overflow); const int if_opcode = iff->Opcode(); - IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); + IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason, overflow ? Op_If : if_opcode); IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); _igvn.hash_delete(lower_bound_iff); lower_bound_iff->set_req(1, lower_bound_bol); @@ -1257,9 +1258,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod } // Test the upper bound - BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow); + BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, range, true, overflow); - IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); + IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason, overflow ? Op_If : if_opcode); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); _igvn.hash_delete(upper_bound_iff); @@ -1272,8 +1273,8 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later // splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.). IfTrueNode* template_assertion_predicate_proj = - add_template_assertion_predicate(iff, loop, hoisted_check_proj, parse_predicate_proj, upper_bound_proj, scale, - offset, init, limit, stride, rng, overflow, reason); + create_template_assertion_predicate(if_opcode, cl, parse_predicate_proj, upper_bound_proj, scale, offset, range, + deopt_reason); // Eliminate the old range check in the loop body. // When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2 @@ -1309,53 +1310,15 @@ void PhaseIdealLoop::eliminate_hoisted_range_check(IfTrueNode* hoisted_check_pro // Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize // them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub // loops are removed properly. -IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, - ParsePredicateSuccessProj* parse_predicate_proj, - IfProjNode* upper_bound_proj, const int scale, Node* offset, - Node* init, Node* limit, const jint stride, - Node* rng, bool& overflow, Deoptimization::DeoptReason reason) { - // First predicate for the initial value on first loop iteration - Node* opaque_init = new OpaqueLoopInitNode(C, init); - register_new_node(opaque_init, upper_bound_proj); - bool negate = (if_proj->_con != parse_predicate_proj->_con); - BoolNode* bol = rc_predicate(upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, - (stride > 0) != (scale > 0), overflow); - Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over - C->add_template_assertion_predicate_opaq(opaque_bol); - register_new_node(opaque_bol, upper_bound_proj); - IfTrueNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(), - false NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); - _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); - assert(opaque_init->outcnt() > 0, "should be used"); - - // Second predicate for init + (current stride - initial stride) - // This is identical to the previous predicate initially but as - // unrolling proceeds current stride is updated. - Node* init_stride = loop->_head->as_CountedLoop()->stride(); - Node* opaque_stride = new OpaqueLoopStrideNode(C, init_stride); - register_new_node(opaque_stride, new_proj); - Node* max_value = new SubINode(opaque_stride, init_stride); - register_new_node(max_value, new_proj); - max_value = new AddINode(opaque_init, max_value); - register_new_node(max_value, new_proj); - // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi - const Type* type_iv = loop->_head->as_CountedLoop()->phi()->bottom_type(); - assert(!type_iv->is_int()->is_con(), "constant indicates one loop iteration for which we bailed out earlier"); - max_value = new CastIINode(new_proj, max_value, type_iv); - register_new_node(max_value, new_proj); - - bol = rc_predicate(new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), - overflow); - opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); - C->add_template_assertion_predicate_opaq(opaque_bol); - register_new_node(opaque_bol, new_proj); - new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(), - false NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); - _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); - assert(max_value->outcnt() > 0, "should be used"); - assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected"); - - return new_proj; +IfTrueNode* PhaseIdealLoop::create_template_assertion_predicate(const int if_opcode, CountedLoopNode* loop_head, + ParsePredicateSuccessProj* parse_predicate_proj, + IfProjNode* new_control, const int scale, Node* offset, + Node* range, Deoptimization::DeoptReason deopt_reason) { + + TemplateAssertionPredicateCreator template_assertion_predicate_creator(loop_head, scale, offset, range, this); + return template_assertion_predicate_creator.create_with_uncommon_trap(new_control, parse_predicate_proj, deopt_reason, + if_opcode); + } // Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 0bed38e5fb068..0689069ed7a71 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1461,12 +1461,13 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) // Create an Initialized Assertion Predicate from the template_assertion_predicate IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, - Node* new_stride, Node* control) { + Node* new_stride, Node* new_control) { assert(assertion_predicate_has_loop_opaque_node(template_assertion_predicate), "must find OpaqueLoop* nodes for Template Assertion Predicate"); - InitializedAssertionPredicateCreator initialized_assertion_predicate(template_assertion_predicate, new_init, - new_stride, this); - IfTrueNode* success_proj = initialized_assertion_predicate.create(control); + InitializedAssertionPredicateCreator initialized_assertion_predicate(this); + IfTrueNode* success_proj = initialized_assertion_predicate.create_from_template(template_assertion_predicate, + new_control, new_init, new_stride); + assert(!assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), "Initialized Assertion Predicates do not have OpaqueLoop* nodes in the bool expression anymore"); return success_proj; @@ -1477,30 +1478,18 @@ IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* templ // We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts, // these are never executed. We therefore insert a Halt node instead of an uncommon trap. Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, - Node* control, IdealLoopTree* outer_loop, Node* input_proj) { + Node* control, IdealLoopTree* outer_loop, Node* new_control) { assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate"); TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4()); assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this); - Node* proj = predicate->clone(); - Node* other_proj = uncommon_proj->clone(); - Node* new_iff = iff->clone(); - new_iff->set_req(1, new_opaque_node); - proj->set_req(0, new_iff); - other_proj->set_req(0, new_iff); - Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr); - register_new_node(frame, C->start()); - Node* halt = new HaltNode(other_proj, frame, "Template Assertion Predicates are always removed before code generation"); - _igvn.add_input_to(C->root(), halt); - new_iff->set_req(0, input_proj); - - register_control(new_iff, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, input_proj); - register_control(proj, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, new_iff); - register_control(other_proj, _ltree_root, new_iff); - register_control(halt, _ltree_root, other_proj); - assert(assertion_predicate_has_loop_opaque_node(proj->in(0)->as_If()), + AssertionPredicateIfCreator assertion_predicate_if_creator(this); + IfTrueNode* success_proj = + assertion_predicate_if_creator.create_for_template(new_control, iff->Opcode(), new_opaque_node + NOT_PRODUCT(COMMA iff->assertion_predicate_type())); + assert(assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); - return proj; + return success_proj; } void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, @@ -2732,40 +2721,6 @@ bool PhaseIdealLoop::is_scaled_iv_plus_extra_offset(Node* exp1, Node* offset3, N return false; } -// Same as PhaseIdealLoop::duplicate_predicates() but for range checks -// eliminated by iteration splitting. -Node* PhaseIdealLoop::add_range_check_elimination_assertion_predicate( - IdealLoopTree* loop, Node* ctrl, const int scale_con, Node* offset, Node* limit, jint stride_con, Node* value, - const bool is_template NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { - bool overflow = false; - BoolNode* bol = rc_predicate(ctrl, scale_con, offset, value, nullptr, stride_con, - limit, (stride_con > 0) != (scale_con > 0), overflow); - Node* opaque_assertion_predicate; - if (is_template) { - opaque_assertion_predicate = new Opaque4Node(C, bol, _igvn.intcon(1)); - } else { - opaque_assertion_predicate = new OpaqueInitializedAssertionPredicateNode(bol, C); - } - register_new_node(opaque_assertion_predicate, ctrl); - IfNode* new_iff = nullptr; - if (overflow) { - new_iff = new IfNode(ctrl, opaque_assertion_predicate, PROB_MAX, COUNT_UNKNOWN); - } else { - new_iff = new RangeCheckNode(ctrl, opaque_assertion_predicate, PROB_MAX, COUNT_UNKNOWN); - } - register_control(new_iff, loop->_parent, ctrl); - Node* iffalse = new IfFalseNode(new_iff); - register_control(iffalse, _ltree_root, new_iff); - ProjNode* iftrue = new IfTrueNode(new_iff); - register_control(iftrue, loop->_parent, new_iff); - Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr); - register_new_node(frame, C->start()); - Node* halt = new HaltNode(iffalse, frame, "range check predicate failed which is impossible"); - register_control(halt, _ltree_root, iffalse); - _igvn.add_input_to(C->root(), halt); - return iftrue; -} - //------------------------------do_range_check--------------------------------- // Eliminate range-checks and other trip-counter vs loop-invariant tests. void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { @@ -2974,45 +2929,39 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { Node* opaque_init = new OpaqueLoopInitNode(C, init); register_new_node(opaque_init, loop_entry); + InitializedAssertionPredicateCreator initialized_assertion_predicate_creator(this); if (abs_stride_is_one) { // If the main loop becomes empty and the array access for this range check is sunk out of the loop, the index // for the array access will be set to the index value of the final iteration which could be out of loop. - // Add an Assertion Predicate for that corner case. The final iv is computed from LoopLimit which is the - // LoopNode::limit() only if abs(stride) == 1 otherwise the computation depends on LoopNode::init_trip() as - // well. When LoopLimit only depends on LoopNode::limit(), there are cases where the zero trip guard for the - // main loop doesn't constant fold after range check elimination but, the array access for the final + // Add an Initialized Assertion Predicate for that corner case. The final iv is computed from LoopLimit which + // is the LoopNode::limit() only if abs(stride) == 1 otherwise the computation depends on LoopNode::init_trip() + // as well. When LoopLimit only depends on LoopNode::limit(), there are cases where the zero trip guard for + // the main loop doesn't constant fold after range check elimination but, the array access for the final // iteration of the main loop is out of bound and the index for that access is out of range for the range // check CastII. - loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, - int_limit, stride_con, final_iv_placeholder, false); + // Note that we do not need to emit a Template Assertion Predicate to update this predicate. When further + // splitting this loop, the final IV will still be the same. When unrolling the loop, we will remove a + // previously added Initialized Assertion Predicate here. But then abs(stride) is greater than 1, and we + // cannot remove an empty loop with a constant limit when init is not a constant as well. We will use + // a LoopLimitCheck node that can only be folded if the zero grip guard is also foldable. + loop_entry = initialized_assertion_predicate_creator.create(final_iv_placeholder, loop_entry, stride_con, + scale_con, int_offset, int_limit NOT_PRODUCT( + COMMA AssertionPredicateType::FinalIv)); assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); } - // Initialized Assertion Predicate for the value of the initial main-loop. - loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, - int_limit, stride_con, init, false); - assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); - // Add two Template Assertion Predicates to create new Initialized Assertion Predicates from when either // unrolling or splitting this main-loop further. - loop_entry = add_range_check_elimination_assertion_predicate( - loop, loop_entry, scale_con, int_offset, int_limit, stride_con, opaque_init, true - NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); + TemplateAssertionPredicateCreator template_assertion_predicate_creator(cl, scale_con , int_offset, int_limit, + this); + loop_entry = template_assertion_predicate_creator.create_with_halt(loop_entry); assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); - Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride()); - register_new_node(opaque_stride, loop_entry); - Node* max_value = new SubINode(opaque_stride, cl->stride()); - register_new_node(max_value, loop_entry); - max_value = new AddINode(opaque_init, max_value); - register_new_node(max_value, loop_entry); - // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi - max_value = new CastIINode(loop_entry, max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); - register_new_node(max_value, loop_entry); - loop_entry = add_range_check_elimination_assertion_predicate( - loop, loop_entry, scale_con, int_offset, int_limit, stride_con, max_value, true - NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); - assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); + // Initialized Assertion Predicate for the value of the initial main-loop. + loop_entry = initialized_assertion_predicate_creator.create(init, loop_entry, stride_con, scale_con, + int_offset, int_limit NOT_PRODUCT(COMMA + AssertionPredicateType::InitValue)); + assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); } else { if (PrintOpto) { @@ -3072,9 +3021,6 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { --imax; } } - - C->print_method(PHASE_AFTER_RANGE_CHECK_ELIMINATION, 4, cl); - } // End of is IF } if (loop_entry != cl->skip_strip_mined()->in(LoopNode::EntryControl)) { @@ -3138,6 +3084,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { set_ctrl(opqzm, new_limit_ctrl); set_ctrl(iffm->in(1)->in(1), new_limit_ctrl); set_ctrl(iffm->in(1), new_limit_ctrl); + + C->print_method(PHASE_AFTER_RANGE_CHECK_ELIMINATION, 4, cl); } // Adjust control for node and its inputs (and inputs of its inputs) to be above the pre end diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 2d169a6459b38..326507bbebb09 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -951,7 +951,7 @@ class PhaseIdealLoop : public PhaseTransform { uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List& old_new); Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, - IdealLoopTree* outer_loop, Node* input_proj); + IdealLoopTree* outer_loop, Node* new_control); IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, Node* control); static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); @@ -1382,20 +1382,17 @@ class PhaseIdealLoop : public PhaseTransform { private: bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero, - Invariance& invar, Deoptimization::DeoptReason reason); + Invariance& invar, Deoptimization::DeoptReason deopt_reason); bool can_create_loop_predicates(const PredicateBlock* profiled_loop_predicate_block) const; bool loop_predication_should_follow_branches(IdealLoopTree* loop, float& loop_trip_cnt); void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, PathFrequency& pf, Node_Stack& stack, VectorSet& seen, Node_List& if_proj_list); - IfTrueNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, - ParsePredicateSuccessProj* parse_predicate_proj, - IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit, - jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason); + IfTrueNode* create_template_assertion_predicate(int if_opcode, CountedLoopNode* loop_head, + ParsePredicateSuccessProj* parse_predicate_proj, + IfProjNode* new_control, int scale, Node* offset, + Node* range, Deoptimization::DeoptReason deopt_reason); void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj); - Node* add_range_check_elimination_assertion_predicate( - IdealLoopTree* loop, Node* predicate_proj, int scale_con, Node* offset, Node* limit, int stride_con, Node* value, - bool is_template NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None)); // Helper function to collect predicate for eliminating the useless ones void eliminate_useless_predicates(); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 18eea3a10bcc6..160bac747ff48 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -23,7 +23,9 @@ */ #include "precompiled.hpp" +#include "opto/addnode.hpp" #include "opto/callnode.hpp" +#include "opto/castnode.hpp" #include "opto/loopnode.hpp" #include "opto/node.hpp" #include "opto/predicates.hpp" @@ -247,11 +249,11 @@ Opaque4Node* TemplateAssertionExpression::clone_and_replace_init(Node* new_init, // Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided // 'new_init' and 'new_stride' nodes, respectively. -Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, - Node* new_ctrl, +Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_control, Node* new_init, + Node* new_stride, PhaseIdealLoop* phase) { ReplaceInitAndStrideStrategy replace_init_and_stride_strategy(new_init, new_stride); - return clone(replace_init_and_stride_strategy, new_ctrl, phase); + return clone(replace_init_and_stride_strategy, new_control, phase); } // Class to collect data nodes from a source to target nodes by following the inputs of the source node recursively. @@ -370,86 +372,293 @@ bool TemplateAssertionExpressionNode::is_template_assertion_predicate(Node* node return node->is_If() && node->in(1)->is_Opaque4(); } -InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(IfNode* template_assertion_predicate, Node* new_init, - Node* new_stride, PhaseIdealLoop* phase) - : _template_assertion_predicate(template_assertion_predicate), - _new_init(new_init), - _new_stride(new_stride), - _phase(phase) {} +// This class creates the Assertion Predicate expression to be used for a Template or Initialized Assertion Predicate. +class AssertionPredicateExpressionCreator : public StackObj { + PhaseIdealLoop* const _phase; + const jint _stride; + const int _scale; + Node* const _offset; + Node* const _range; + const bool _upper; -// Create an Initialized Assertion Predicate at the provided control from the _template_assertion_predicate. -// We clone the Template Assertion Expression and replace: -// - Opaque4 with OpaqueInitializedAssertionPredicate -// - OpaqueLoop*Nodes with _new_init and _new_stride, respectively. + public: + AssertionPredicateExpressionCreator(const int stride, const int scale, Node* offset, Node* range, + PhaseIdealLoop* phase) + : _phase(phase), + _stride(stride), + _scale(scale), + _offset(offset), + _range(range), + _upper((_stride > 0) != (_scale > 0)) {} // Make sure rc_predicate() chooses the "scale*init + offset" case. + + // Create the expression for a Template Assertion Predicate with an Opaque4 node. + Opaque4Node* create_for_template(Node* new_control, Node* operand, bool& does_overflow) const { + BoolNode* bool_for_expression = _phase->rc_predicate(new_control, _scale, _offset, operand, nullptr, + _stride, _range, _upper, does_overflow); + return create_opaque4_node(new_control, bool_for_expression); + } + + private: + Opaque4Node* create_opaque4_node(Node* new_control, BoolNode* bool_for_expression) const { + Compile* C = _phase->C; + Opaque4Node* new_expression = new Opaque4Node(C, bool_for_expression, _phase->igvn().intcon(1)); + C->add_template_assertion_predicate_opaq(new_expression); + _phase->register_new_node(new_expression, new_control); + return new_expression; + } + + public: + // Create the expression for an Initialized Assertion Predicate with an OpaqueInitializedAssertionPredicate node. + OpaqueInitializedAssertionPredicateNode* create_for_initialized(Node* new_control, Node* operand, + bool& does_overflow) const { + BoolNode* bool_for_expression = _phase->rc_predicate(new_control, _scale, _offset, operand, nullptr, + _stride, _range, _upper, does_overflow); + return create_opaque_initialized_assertion_predicate_node(new_control, bool_for_expression); + } + + private: + OpaqueInitializedAssertionPredicateNode* create_opaque_initialized_assertion_predicate_node( + Node* new_control, BoolNode* bool_for_expression) const { + OpaqueInitializedAssertionPredicateNode* new_expression = + new OpaqueInitializedAssertionPredicateNode(bool_for_expression, _phase->C); + _phase->register_new_node(new_expression, new_control); + return new_expression; + } +}; + +IfTrueNode* AssertionPredicateIfCreator::create_for_template(Node* new_control, const int if_opcode, + Node* assertion_expression NOT_PRODUCT(COMMA + const AssertionPredicateType assertion_predicate_type)) { + const char* halt_message = "Template Assertion Predicates are always removed before code generation"; + return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type)); +} + +IfTrueNode* AssertionPredicateIfCreator::create_for_initialized(Node* new_control, const int if_opcode, + Node* assertion_expression NOT_PRODUCT(COMMA + const AssertionPredicateType assertion_predicate_type)) { + const char* halt_message = "Initialized Assertion Predicate cannot fail"; + return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type)); +} + +// Creates the If node for an Assertion Predicate with a success path and a fail path having a Halt node: // -// / init stride -// | | | -// | OpaqueLoopInitNode OpaqueLoopStrideNode / _new_init _new_stride -// Template | \ / | \ / -// Assertion | ... Assertion | ... -// Expression | | Expression | | -// | Bool | new Bool -// | | | | -// \ Opaque4 ======> control \ OpaqueInitializedAssertionPredicate -// | \ / -// If new If -// / \ / \ -// success fail path new success new Halt -// proj (Halt or UCT) proj +// new_control assertion_expression +// \ / +// If +// / \ +// success fail path +// proj with Halt // -IfTrueNode* InitializedAssertionPredicateCreator::create(Node* control) { - IdealLoopTree* loop = _phase->get_loop(control); - OpaqueInitializedAssertionPredicateNode* assertion_expression = create_assertion_expression(control); - IfNode* if_node = create_if_node(control, assertion_expression, loop); - create_fail_path(if_node, loop); +IfTrueNode* AssertionPredicateIfCreator::create(Node* new_control, const int if_opcode, Node* assertion_expression, + const char* halt_message NOT_PRODUCT(COMMA + const AssertionPredicateType assertion_predicate_type)) { + assert(assertion_expression->is_Opaque4() || assertion_expression->is_OpaqueInitializedAssertionPredicate(), + "not a valid assertion expression"); + IdealLoopTree* loop = _phase->get_loop(new_control); + IfNode* if_node = create_if_node(new_control, if_opcode, assertion_expression, loop + NOT_PRODUCT(COMMA assertion_predicate_type)); + create_fail_path(if_node, loop, halt_message); return create_success_path(if_node, loop); } -// Create a new Assertion Expression to be used as bool input for the Initialized Assertion Predicate IfNode. -OpaqueInitializedAssertionPredicateNode* InitializedAssertionPredicateCreator::create_assertion_expression(Node* control) { - Opaque4Node* template_opaque = _template_assertion_predicate->in(1)->as_Opaque4(); - TemplateAssertionExpression template_assertion_expression(template_opaque); - Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(_new_init, _new_stride, - control, _phase); - OpaqueInitializedAssertionPredicateNode* assertion_expression = - new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C); - _phase->register_new_node(assertion_expression, control); - return assertion_expression; -} - -IfNode* InitializedAssertionPredicateCreator::create_if_node(Node* control, - OpaqueInitializedAssertionPredicateNode* assertion_expression, - IdealLoopTree* loop) { - const int if_opcode = _template_assertion_predicate->Opcode(); - NOT_PRODUCT(const AssertionPredicateType assertion_predicate_type = _template_assertion_predicate->assertion_predicate_type();) - IfNode* if_node = if_opcode == Op_If ? - new IfNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type)) : - new RangeCheckNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type)); - _phase->register_control(if_node, loop, control); +IfNode* AssertionPredicateIfCreator::create_if_node(Node* new_control, const int if_opcode, Node* assertion_expression, + IdealLoopTree* loop NOT_PRODUCT(COMMA + const AssertionPredicateType assertion_predicate_type)) { + IfNode* if_node; + if (if_opcode == Op_If) { + if_node = new IfNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN + NOT_PRODUCT(COMMA assertion_predicate_type)); + } else { + assert(if_opcode == Op_RangeCheck, "must be range check"); + if_node = new RangeCheckNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN + NOT_PRODUCT(COMMA assertion_predicate_type)); + } + _phase->register_control(if_node, loop, new_control); return if_node; } -IfTrueNode* InitializedAssertionPredicateCreator::create_success_path(IfNode* if_node, IdealLoopTree* loop) { +IfTrueNode* AssertionPredicateIfCreator::create_success_path(IfNode* if_node, IdealLoopTree* loop) { IfTrueNode* success_proj = new IfTrueNode(if_node); _phase->register_control(success_proj, loop, if_node); return success_proj; } -void InitializedAssertionPredicateCreator::create_fail_path(IfNode* if_node, IdealLoopTree* loop) { +void AssertionPredicateIfCreator::create_fail_path(IfNode* if_node, IdealLoopTree* loop, const char* halt_message) { IfFalseNode* fail_proj = new IfFalseNode(if_node); _phase->register_control(fail_proj, loop, if_node); - create_halt_node(fail_proj, loop); + create_halt_node(fail_proj, loop, halt_message); } -void InitializedAssertionPredicateCreator::create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop) { +void AssertionPredicateIfCreator::create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, + const char* halt_message) { StartNode* start_node = _phase->C->start(); Node* frame = new ParmNode(start_node, TypeFunc::FramePtr); _phase->register_new_node(frame, start_node); - Node* halt = new HaltNode(fail_proj, frame, "Initialized Assertion Predicate cannot fail"); + Node* halt = new HaltNode(fail_proj, frame, halt_message); _phase->igvn().add_input_to(_phase->C->root(), halt); _phase->register_control(halt, loop, fail_proj); } +// Creates an init and last value Template Assertion Predicate from a Parse Predicate with an UCT on the failing path. +// Returns the success projection of the latter. +IfTrueNode* TemplateAssertionPredicateCreator::create_with_uncommon_trap(Node* new_control, + ParsePredicateSuccessProj* parse_predicate_success_proj, + const Deoptimization::DeoptReason deopt_reason, + const int if_opcode) { + OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control); + bool does_overflow; + Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init, + does_overflow); + IfTrueNode* template_predicate_success_proj = + create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj, + deopt_reason, if_opcode, does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); + template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init, + does_overflow); + return create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj, + deopt_reason, if_opcode, does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); +} + +OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* new_control) { + OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _loop_head->init_trip()); + _phase->register_new_node(opaque_init, new_control); + return opaque_init; +} + +Opaque4Node* TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const { + AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase); + return expression_creator.create_for_template(new_control, opaque_init, does_overflow); +} + +IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap( + Opaque4Node* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj, + const Deoptimization::DeoptReason deopt_reason, const int if_opcode, const bool does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + IfTrueNode* success_proj = _phase->create_new_if_for_predicate(parse_predicate_success_proj, nullptr, deopt_reason, + does_overflow ? Op_If : if_opcode, false + NOT_PRODUCT(COMMA assertion_predicate_type)); + _phase->igvn().replace_input_of(success_proj->in(0), 1, template_assertion_predicate_expression); + return success_proj; +} + +Opaque4Node* TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, + bool& does_overflow) const { + Node* last_value = create_last_value(new_control, opaque_init); + AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase); + return expression_creator.create_for_template(new_control, last_value, does_overflow); +} + +Node* TemplateAssertionPredicateCreator::create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const { + Node* init_stride = _loop_head->stride(); + Node* opaque_stride = new OpaqueLoopStrideNode(_phase->C, init_stride); + _phase->register_new_node(opaque_stride, new_control); + Node* last_value = new SubINode(opaque_stride, init_stride); + _phase->register_new_node(last_value, new_control); + last_value = new AddINode(opaque_init, last_value); + _phase->register_new_node(last_value, new_control); + // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv phi + last_value = new CastIINode(new_control, last_value, _loop_head->phi()->bottom_type()); + _phase->register_new_node(last_value, new_control); + return last_value; +} + +IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt( + Node* new_control, Opaque4Node* template_assertion_predicate_expression, bool does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); + return assertion_predicate_if_creator.create_for_template(new_control, does_overflow ? Op_If : Op_RangeCheck, + template_assertion_predicate_expression + NOT_PRODUCT(COMMA assertion_predicate_type)); +} + +// Creates an init and last value Template Assertion Predicate connected together and returns the success projection +// of the latter. +IfTrueNode* TemplateAssertionPredicateCreator::create_with_halt(Node* new_control) { + OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control); + bool does_overflow; + Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init, + does_overflow); + IfTrueNode* template_predicate_success_proj = + create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); + template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init, + does_overflow); + return create_if_node_with_halt(template_predicate_success_proj, template_assertion_predicate_expression, + does_overflow NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); +} + +InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(PhaseIdealLoop* phase) + : _phase(phase) {} + +// Create an Initialized Assertion Predicate from the provided template_assertion_predicate at 'new_control'. +// We clone the Template Assertion Expression and replace: +// - Opaque4 with OpaqueInitializedAssertionPredicate +// - OpaqueLoop*Nodes with new_init and _ew_stride, respectively. +// +// / init stride +// | | | +// | OpaqueLoopInitNode OpaqueLoopStrideNode / new_init new_stride +// Template | \ / | \ / +// Assertion | ... Assertion | ... +// Expression | | Expression | | +// | Bool | new Bool +// | | | | +// \ Opaque4 ======> new_control \ OpaqueInitializedAssertionPredicate +// | \ / +// If new If +// / \ / \ +// success fail path new success new Halt +// proj (Halt or UCT) proj +// +IfTrueNode* InitializedAssertionPredicateCreator::create_from_template(IfNode* template_assertion_predicate, + Node* new_control, Node* new_init, + Node* new_stride) { + OpaqueInitializedAssertionPredicateNode* assertion_expression = + create_assertion_expression_from_template(template_assertion_predicate, new_control, new_init, new_stride); + return create_control_nodes(new_control, template_assertion_predicate->Opcode(), assertion_expression + NOT_PRODUCT(COMMA template_assertion_predicate->assertion_predicate_type())); +} + +// Create a new Initialized Assertion Predicate directly without a template. +IfTrueNode* InitializedAssertionPredicateCreator::create(Node* operand, Node* new_control, const jint stride, + const int scale, Node* offset, Node* range NOT_PRODUCT(COMMA + AssertionPredicateType assertion_predicate_type)) { + AssertionPredicateExpressionCreator expression_creator(stride, scale, offset, range, _phase); + bool does_overflow; + OpaqueInitializedAssertionPredicateNode* assertion_expression = + expression_creator.create_for_initialized(new_control, operand, does_overflow); + return create_control_nodes(new_control, does_overflow ? Op_If : Op_RangeCheck, assertion_expression + NOT_PRODUCT(COMMA assertion_predicate_type)); +} + +// Creates the CFG nodes for the Initialized Assertion Predicate. +IfTrueNode* InitializedAssertionPredicateCreator::create_control_nodes( + Node* new_control, const int if_opcode, OpaqueInitializedAssertionPredicateNode* assertion_expression + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); + return assertion_predicate_if_creator.create_for_initialized(new_control, if_opcode, assertion_expression + NOT_PRODUCT(COMMA assertion_predicate_type)); +} + +// Create a new Assertion Expression based from the given template to be used as bool input for the Initialized +// Assertion Predicate IfNode. +OpaqueInitializedAssertionPredicateNode* +InitializedAssertionPredicateCreator::create_assertion_expression_from_template(IfNode* template_assertion_predicate, + Node* new_control, Node* new_init, + Node* new_stride) { + Opaque4Node* template_opaque = template_assertion_predicate->in(1)->as_Opaque4(); + TemplateAssertionExpression template_assertion_expression(template_opaque); + Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(new_control, new_init, + new_stride, + _phase); + OpaqueInitializedAssertionPredicateNode* assertion_expression = + new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C); + _phase->register_new_node(assertion_expression, new_control); + return assertion_expression; +} + #ifndef PRODUCT void PredicateBlock::dump() const { dump(""); diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index b38b888cc3dba..b487c9a4d9394 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -207,7 +207,9 @@ class TemplateAssertionPredicate; enum class AssertionPredicateType { None, // Not an Assertion Predicate InitValue, - LastValue + LastValue, + // Used for the Initialized Assertion Predicate emitted during Range Check Elimination for the final IV value. + FinalIv }; #endif // NOT PRODUCT @@ -442,8 +444,9 @@ class TemplateAssertionExpression : public StackObj { public: Opaque4Node* clone(Node* new_ctrl, PhaseIdealLoop* phase); - Opaque4Node* clone_and_replace_init(Node* new_init, Node* new_ctrl,PhaseIdealLoop* phase); - Opaque4Node* clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, Node* new_ctrl, PhaseIdealLoop* phase); + Opaque4Node* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase); + Opaque4Node* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride, + PhaseIdealLoop* phase); }; // Class to represent a node being part of a Template Assertion Expression. Note that this is not an IR node. @@ -520,26 +523,85 @@ class TemplateAssertionExpressionNode : public StackObj { } }; -// This class creates a new Initialized Assertion Predicate. + +// This class is used to create the actual If node with a success path and a fail path with a Halt node. +class AssertionPredicateIfCreator : public StackObj { + PhaseIdealLoop* const _phase; + + public: + explicit AssertionPredicateIfCreator(PhaseIdealLoop* const phase) : _phase(phase) {} + NONCOPYABLE(AssertionPredicateIfCreator); + + IfTrueNode* create_for_initialized(Node* new_control, int if_opcode, Node* assertion_expression + NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create_for_template(Node* new_control, int if_opcode, Node* assertion_expression + NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + private: + IfTrueNode* create(Node* new_control, int if_opcode, Node* assertion_expression, const char* halt_message + NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + IfNode* create_if_node(Node* new_control, int if_opcode, Node* assertion_expression, IdealLoopTree* loop + NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop); + void create_fail_path(IfNode* if_node, IdealLoopTree* loop, const char* halt_message); + void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, const char* halt_message); +}; + +// This class is used to create a Template Assertion Predicate either with an UCT or a Halt Node from scratch. +class TemplateAssertionPredicateCreator : public StackObj { + CountedLoopNode* const _loop_head; + const int _scale; + Node* const _offset; + Node* const _range; + PhaseIdealLoop* const _phase; + + OpaqueLoopInitNode* create_opaque_init(Node* new_control); + Opaque4Node* create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const; + Opaque4Node* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const; + Node* create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const; + IfTrueNode* create_if_node_with_uncommon_trap(Opaque4Node* template_assertion_predicate_expression, + ParsePredicateSuccessProj* parse_predicate_success_proj, + Deoptimization::DeoptReason deopt_reason, int if_opcode, + bool does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create_if_node_with_halt(Node* new_control, Opaque4Node* template_assertion_predicate_expression, + bool does_overflow + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + + public: + TemplateAssertionPredicateCreator(CountedLoopNode* loop_head, int scale, Node* offset, Node* range, + PhaseIdealLoop* phase) + : _loop_head(loop_head), + _scale(scale), + _offset(offset), + _range(range), + _phase(phase) {} + NONCOPYABLE(TemplateAssertionPredicateCreator); + + IfTrueNode* create_with_uncommon_trap(Node* new_control, ParsePredicateSuccessProj* parse_predicate_success_proj, + Deoptimization::DeoptReason deopt_reason, int if_opcode); + IfTrueNode* create_with_halt(Node* new_control); +}; + +// This class creates a new Initialized Assertion Predicate either from a template or from scratch. class InitializedAssertionPredicateCreator : public StackObj { - IfNode* const _template_assertion_predicate; - Node* const _new_init; - Node* const _new_stride; PhaseIdealLoop* const _phase; public: - InitializedAssertionPredicateCreator(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, - PhaseIdealLoop* phase); + explicit InitializedAssertionPredicateCreator(PhaseIdealLoop* phase); NONCOPYABLE(InitializedAssertionPredicateCreator); - IfTrueNode* create(Node* control); + IfTrueNode* create_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init, + Node* new_stride); + IfTrueNode* create(Node* operand, Node* new_control, jint stride, int scale, Node* offset, Node* range + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); private: - OpaqueInitializedAssertionPredicateNode* create_assertion_expression(Node* control); - IfNode* create_if_node(Node* control, OpaqueInitializedAssertionPredicateNode* assertion_expression, IdealLoopTree* loop); - void create_fail_path(IfNode* if_node, IdealLoopTree* loop); - void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop); - IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop); + OpaqueInitializedAssertionPredicateNode* create_assertion_expression_from_template(IfNode* template_assertion_predicate, + Node* new_control, Node* new_init, + Node* new_stride); + IfTrueNode* create_control_nodes(Node* new_control, int if_opcode, + OpaqueInitializedAssertionPredicateNode* assertion_expression + NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); }; // This class iterates through all predicates of a Regular Predicate Block and applies the given visitor to each.